From a8ce7ab1f323c40d79af7343cf8e2d887930a492 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 8 Sep 2023 17:11:39 +0200 Subject: [PATCH] DM-285: Entity to component mappings with new ruleset * refactoring structure * added custom syntax validation, such that rule files may not be corrupted * added component drools preparations * deprecated redactionLog * integrated entityLog * integrated new getRules with ruleFileType * renamed service.graph package -> service.document * fixed tests * update layout parser version * initial component drools file - wip Signed-off-by: Kilian Schuettler --- .../build.gradle.kts | 6 +- .../v1/server/model/ManualEntity.java | 19 +- .../v1/server/model/component/Component.java | 41 ++ .../v1/server/model/component/Entity.java | 106 ++++ .../model/document/entity/Component.java | 24 - .../entity/{Entity.java => IEntity.java} | 24 +- .../model/document/entity/TextEntity.java | 7 +- .../v1/server/model/document/nodes/Image.java | 39 +- .../server/model/document/nodes/NodeType.java | 2 +- .../v1/server/service/AnalyzeService.java | 138 ++--- .../service/ComponentLogCreatorService.java | 71 +-- .../service/EntityChangeLogService.java | 116 ++++ .../service/EntityLogCreatorService.java | 18 +- .../ManualChangesApplicationService.java | 8 +- .../document/ComponentCreationService.java | 126 ++++ .../service/document/EntityComparators.java | 41 ++ .../{ => document}/SectionFinderService.java | 20 +- .../ComponentDroolsExecutionService.java | 78 ++- .../storage/RedactionStorageService.java | 10 + .../v1/server/utils/DateConverter.java | 49 ++ .../v1/server/DocumineFloraTest.java | 4 +- .../v1/server/annotate/AnnotationService.java | 9 +- ...umentIEntityInsertionIntegrationTest.java} | 2 +- .../DocumentPerformanceIntegrationTest.java | 2 +- .../DocumentVisualizationIntegrationTest.java | 2 +- .../DroolsSyntaxValidationServiceTest.java | 9 + .../adapter/NerEntitiesAdapterTest.java | 4 +- .../utils/LayoutParsingRequestProvider.java | 3 +- .../drools/documine_flora_components.drl | 558 ++++++++++++++++++ .../src/test/resources/drools/rules.drl | 1 + .../src/test/resources/drools/rules_v2.drl | 1 + .../EFSA_sanitisation_GFL_v1/rules.drl | 2 +- 32 files changed, 1346 insertions(+), 194 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/{Entity.java => IEntity.java} (94%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/{ => document}/SectionFinderService.java (93%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java rename redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/{DocumentEntityInsertionIntegrationTest.java => DocumentIEntityInsertionIntegrationTest.java} (99%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 68a1a141..7608ff3e 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -12,11 +12,11 @@ plugins { description = "redaction-service-server-v1" -val layoutParserVersion = "0.25.0" +val layoutParserVersion = "0.70.0" val jacksonVersion = "2.15.2" val droolsVersion = "8.43.0.Final" -val pdfBoxVersion = "3.0.0-alpha2" -val persistenceServiceVersion = "DM-285-3" +val pdfBoxVersion = "3.0.0" +val persistenceServiceVersion = "DM-285-5" configurations { all { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java index 617729e5..e3104457 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java @@ -5,7 +5,8 @@ import java.util.PriorityQueue; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; @@ -20,7 +21,7 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class ManualEntity implements Entity { +public class ManualEntity implements IEntity { // The id must be mapped into a TextEntity as is for comments to work correctly String id; @@ -88,4 +89,18 @@ public class ManualEntity implements Entity { .build(); } + + @Override + public TextRange getTextRange() { + + return new TextRange(-1, -1); + } + + + @Override + public String type() { + + return getManualOverwrite().getType().orElse(type); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java new file mode 100644 index 00000000..d820e712 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.redaction.v1.server.model.component; + +import java.util.Collection; +import java.util.List; + +import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class Component { + + RuleIdentifier matchedRule; + String category; + String value; + String transformation; + + List references; + + + public boolean addReference(Entity entity) { + + return references.add(entity); + } + + + public boolean addReferences(Collection entities) { + + return references.addAll(entities); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java new file mode 100644 index 00000000..035289dc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java @@ -0,0 +1,106 @@ +package com.iqser.red.service.redaction.v1.server.model.component; + +import java.util.List; +import java.util.Set; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.experimental.FieldDefaults; + +@Getter +@Builder +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class Entity { + + String id; + String type; + EntryType entryType; + EntryState state; + String value; + String reason; + String matchedRule; + boolean rectangle; + String legalBasis; + + boolean imported; + + String section; + float[] color; + + List positions; + int sectionNumber; + + String textBefore; + String textAfter; + + int startOffset; + int endOffset; + int length; + + boolean imageHasTransparency; + + boolean isDictionaryEntry; + boolean isDossierDictionaryEntry; + + boolean excluded; + + String sourceId; + + List changes; + + List manualChanges; + + Set engines; + + Set reference; + + Set importedRedactionIntersections; + + + public static Entity fromEntityLogEntry(EntityLogEntry e) { + + return Entity.builder() + .id(e.getId()) + .type(e.getType()) + .entryType(e.getEntryType()) + .state(e.getState()) + .value(e.getValue()) + .reason(e.getReason()) + .matchedRule(e.getMatchedRule()) + .rectangle(e.isRectangle()) + .legalBasis(e.getLegalBasis()) + .imported(e.isImported()) + .section(e.getSection()) + .color(e.getColor()) + .positions(e.getPositions()) + .sectionNumber(e.getSectionNumber()) + .textBefore(e.getTextBefore()) + .textAfter(e.getTextAfter()) + .startOffset(e.getStartOffset()) + .endOffset(e.getEndOffset()) + .length(e.getValue().length()) + .imageHasTransparency(e.isImageHasTransparency()) + .isDictionaryEntry(e.isDictionaryEntry()) + .isDossierDictionaryEntry(e.isDossierDictionaryEntry()) + .excluded(e.isExcluded()) + .sourceId(e.getSourceId()) + .changes(e.getChanges()) + .manualChanges(e.getManualChanges()) + .engines(e.getEngines()) + .reference(e.getReference()) + .importedRedactionIntersections(e.getImportedRedactionIntersections()) + .build(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java deleted file mode 100644 index 98fce44f..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.model.document.entity; - -import java.util.List; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class Component { - - String value; - String originalValue; - String transformation; - - List references; -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java similarity index 94% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Entity.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java index b9e2a005..9d43e342 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java @@ -5,11 +5,12 @@ import java.util.HashSet; import java.util.PriorityQueue; import java.util.Set; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; import lombok.NonNull; -public interface Entity { +public interface IEntity { PriorityQueue getMatchedRuleList(); @@ -17,6 +18,27 @@ public interface Entity { ManualChangeOverwrite getManualOverwrite(); + String getValue(); + + + TextRange getTextRange(); + + + String type(); + + + default int length() { + + return value().length(); + } + + + default String value() { + + return getManualOverwrite().getValue().orElse(getValue()); + } + + // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner. default boolean applied() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index cace9fb2..1110f7b7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -28,7 +28,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) @EqualsAndHashCode(onlyExplicitlyIncluded = true) -public class TextEntity implements Entity { +public class TextEntity implements IEntity { // primary key @EqualsAndHashCode.Include @@ -201,5 +201,10 @@ public class TextEntity implements Entity { } + @Override + public String type() { + + return getManualOverwrite().getType().orElse(type); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java index fd584387..2f0c8a2e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java @@ -5,12 +5,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.PriorityQueue; import java.util.Set; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; @@ -30,7 +32,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class Image implements GenericSemanticNode, Entity { +public class Image implements GenericSemanticNode, IEntity { List treeId; String id; @@ -77,6 +79,20 @@ public class Image implements GenericSemanticNode, Entity { } + @Override + public TextRange getTextRange() { + + return GenericSemanticNode.super.getTextRange(); + } + + + @Override + public String type() { + + return getManualOverwrite().getType().orElse(imageType.toString()); + } + + @Override public String toString() { @@ -92,4 +108,23 @@ public class Image implements GenericSemanticNode, Entity { return bBoxPerPage; } + + @Override + public String getValue() { + + return NodeType.IMAGE + ":" + camelCase(imageType.toString()); + } + + + private String camelCase(String name) { + + return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH); + } + + + public int length() { + + return 0; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java index 0c06dfa4..8fe8a291 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java @@ -16,6 +16,6 @@ public enum NodeType { public String toString() { - return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT); + return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ENGLISH); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 16e7dd56..e87db7eb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; @@ -35,15 +36,16 @@ import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService; -import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; -import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter; +import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService; +import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import io.micrometer.core.annotation.Timed; @@ -71,6 +73,7 @@ public class AnalyzeService { ComponentLogCreatorService componentLogCreatorService; RedactionStorageService redactionStorageService; RedactionChangeLogService redactionChangeLogService; + EntityChangeLogService entityChangeLogService; LegalBasisClient legalBasisClient; RedactionServiceSettings redactionServiceSettings; ImportedRedactionService importedRedactionService; @@ -114,27 +117,14 @@ public class AnalyzeService { nerEntities); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - ComponentLog componentLog = new ComponentLog(); - if (kieWrapperComponentRules.isPresent()) { - List components = componentDroolsExecutionService.executeRules(document.getEntities(), - allFileAttributes, - notFoundManualRedactionEntries, - document.streamAllImages().toList()); - - componentLog = componentLogCreatorService.buildComponentLog(kieWrapperComponentRules.container(), analyzeRequest.getAnalysisNumber(), components); - log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - } - RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); return finalizeAnalysis(analyzeRequest, - startTime, - entityLog, componentLog, + startTime, kieWrapperComponentRules, entityLog, redactionLog, document.getNumberOfPages(), - dictionary.getVersion(), - false, new HashSet<>(allFileAttributes)); + dictionary.getVersion(), false, new HashSet<>(allFileAttributes)); } @@ -184,16 +174,6 @@ public class AnalyzeService { log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); ComponentLog previousComponentLog = new ComponentLog(); - KieWrapper kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT); - if (kieWrapperComponentRules.isPresent()) { - log.info("Updated component rules to version {} for file {} in dossier {}", - kieWrapperComponentRules.rulesVersion(), - analyzeRequest.getFileId(), - analyzeRequest.getDossierId()); - previousComponentLog = redactionStorageService.getComponentLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - log.info("Loaded previous component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - } - Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -203,30 +183,28 @@ public class AnalyzeService { } DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), - new DictionaryVersion(previousRedactionLog.getDictionaryVersion(), previousRedactionLog.getDossierDictionaryVersion()), + new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement); + Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement); List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (sectionsToReAnalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, - startTime, - previousEntityLog, previousComponentLog, + startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), previousEntityLog, previousRedactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, Collections.emptySet()); } + KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); + log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); - log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); @@ -244,23 +222,11 @@ public class AnalyzeService { nerEntities); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - ComponentLog componentLog = new ComponentLog(); - if (kieWrapperComponentRules.isPresent()) { - List components = componentDroolsExecutionService.executeRules(document.getEntities(), - allFileAttributes, - notFoundManualRedactionEntries, - document.streamAllImages().toList()); - - componentLog = componentLogCreatorService.buildComponentLog(kieWrapperComponentRules.container(), analyzeRequest.getAnalysisNumber(), components); - log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - } - RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds); return finalizeAnalysis(analyzeRequest, - startTime, - entityLog, componentLog, + startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLog, redactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -316,26 +282,20 @@ public class AnalyzeService { private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, - long startTime, - EntityLog entityLog, ComponentLog componentLog, + long startTime, KieWrapper kieWrapperComponentRules, EntityLog entityLog, RedactionLog redactionLog, int numberOfPages, DictionaryVersion dictionaryVersion, boolean isReanalysis, Set addedFileAttributes) { - redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); - redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); - excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); + EntityLogChanges entityLogChanges = finalizeEntityLog(analyzeRequest, entityLog, redactionLog, dictionaryVersion); - RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - redactionLog, - analyzeRequest.getAnalysisNumber()); - log.info("Created Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + if (entityLogChanges.isHasChanges()) { + computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges); + } - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); - log.info("Stored Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); long duration = System.currentTimeMillis() - startTime; @@ -346,13 +306,13 @@ public class AnalyzeService { .fileId(analyzeRequest.getFileId()) .duration(duration) .numberOfPages(numberOfPages) - .hasUpdates(redactionLogChange.isHasChanges()) + .hasUpdates(entityLogChanges.isHasChanges()) .analysisVersion(redactionServiceSettings.getAnalysisVersion()) .analysisNumber(analyzeRequest.getAnalysisNumber()) - .rulesVersion(redactionLog.getRulesVersion()) - .dictionaryVersion(redactionLog.getDictionaryVersion()) - .legalBasisVersion(redactionLog.getLegalBasisVersion()) - .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion()) + .rulesVersion(entityLog.getRulesVersion()) + .dictionaryVersion(entityLog.getDictionaryVersion()) + .legalBasisVersion(entityLog.getLegalBasisVersion()) + .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion()) .wasReanalyzed(isReanalysis) .manualRedactions(analyzeRequest.getManualRedactions()) .addedFileAttributes(addedFileAttributes) @@ -360,16 +320,58 @@ public class AnalyzeService { } + private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, + KieWrapper kieWrapperComponentRules, + Set addedFileAttributes, + EntityLogChanges entityLogChanges) { + + if (!kieWrapperComponentRules.isPresent()) { + return; + } + + List components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), + entityLogChanges.getEntityLog(), + addedFileAttributes.stream().toList()); + log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components); + + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog); + + } + + + private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, EntityLog entityLog, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) { + + redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); + redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); + excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); + EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + entityLog, + analyzeRequest.getAnalysisNumber()); + RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + redactionLog, + analyzeRequest.getAnalysisNumber()); + log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog()); + return entityLogChanges; + } + + private static List getSectionsToReAnalyse(Document document, Set sectionsToReanalyseIds) { return document.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTreeId().get(0))).collect(Collectors.toList()); } - private Set getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, Document document, DictionaryIncrement dictionaryIncrement) { + private Set getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) { return analyzeRequest.getSectionsToReanalyse().isEmpty() // - ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, redactionLog, document, analyzeRequest) // + ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) // : analyzeRequest.getSectionsToReanalyse(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java index ee6df49e..83a500fa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java @@ -2,28 +2,30 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.List; import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.kie.api.runtime.KieContainer; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Component; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators; @Service public class ComponentLogCreatorService { - public ComponentLog buildComponentLog(KieContainer container, int analysisNumber, List components) { + public ComponentLog buildComponentLog(int analysisNumber, List components) { - List componentLogEntries = components.stream().map(this::buildComponentLogEntry).toList(); - return new ComponentLog(analysisNumber, componentLogEntries, -1, -1, -1); + List componentLogCategories = components.stream() + .collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList()))) + .entrySet() + .stream() + .map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue())) + .toList(); + return new ComponentLog(analysisNumber, componentLogCategories, -1, -1, -1); } @@ -31,8 +33,7 @@ public class ComponentLogCreatorService { return ComponentLogEntry.builder() .value(component.getValue()) - .transformation(component.getTransformation()) - .originalValue(component.getOriginalValue()) + .transformation(component.getTransformation()).originalValue(component.getReferences().stream().sorted(EntityComparators.start()).map(Entity::getValue).toList()) .componentEntityReferences(toComponentEntityReferences(component.getReferences())) .build(); } @@ -40,45 +41,19 @@ public class ComponentLogCreatorService { private List toComponentEntityReferences(List references) { - return references.stream().flatMap(this::toComponentEntityReference).toList(); + return references.stream().map(this::toComponentEntityReference).toList(); } - private Stream toComponentEntityReference(Entity entity) { + private ComponentEntityReference toComponentEntityReference(Entity entity) { - if (entity instanceof TextEntity textEntity) { - textEntity.getPositionsOnPagePerPage() - .stream() - .map(positionOnPage -> ComponentEntityReference.builder() - .page(positionOnPage.getPage().getNumber()) - .type(textEntity.getType()) - .id(positionOnPage.getId()) - .ruleIdentifier(textEntity.getMatchedRule().getRuleIdentifier().toString()) - .reason(textEntity.buildReasonWithManualChangeDescriptions()) - .build()); - } else if (entity instanceof Image image) { - return Stream.of(ComponentEntityReference.builder() - .id(image.getId()) - .reason(image.buildReasonWithManualChangeDescriptions()) - .type(image.getImageType().toString()) - .ruleIdentifier(image.getMatchedRule().getRuleIdentifier().toString()) - .page(image.getFirstPage().getNumber()) - .build()); - } else if (entity instanceof ManualEntity manualEntity) { - return manualEntity.getEntityPosition() - .stream() - .collect(Collectors.groupingBy(RectangleWithPage::pageNumber)) - .keySet() - .stream() - .map(pageNumber -> ComponentEntityReference.builder() - .id(manualEntity.getId()) - .reason(manualEntity.buildReasonWithManualChangeDescriptions()) - .type(manualEntity.getType()) - .ruleIdentifier(manualEntity.getRuleIdentifier()) - .page(pageNumber) - .build()); - } - throw new UnsupportedOperationException(String.format("Entity with class %s is not implemented yet", entity.getClass())); + return ComponentEntityReference.builder() + .id(entity.getId()) + .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)) + .reason(entity.getReason()) + .ruleIdentifier(entity.getMatchedRule()) + .type(entity.getType()) + .build(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java new file mode 100644 index 00000000..8262b4ce --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -0,0 +1,116 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class EntityChangeLogService { + + private final RedactionStorageService redactionStorageService; + + + @Timed("redactmanager_computeChanges") + public EntityLogChanges computeChanges(String dossierId, String fileId, EntityLog currentEntityLog, int analysisNumber) { + + long start = System.currentTimeMillis(); + + EntityLog previousEntityLog = redactionStorageService.getEntityLog(dossierId, fileId); + + if (previousEntityLog == null) { + currentEntityLog.getEntityLogEntry().forEach(entry -> { + entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); + }); + return new EntityLogChanges(currentEntityLog, false); + } + + List previouslyExistingEntries = previousEntityLog.getEntityLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList(); + + Map addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousEntityLog(currentEntityLog, previouslyExistingEntries); + Set removedIds = getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(currentEntityLog, previouslyExistingEntries); + + List newEntityLogEntries = previousEntityLog.getEntityLogEntry(); + + List toRemove = new ArrayList<>(); + newEntityLogEntries.forEach(entry -> { + if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now())); + var newEntry = addedEntryIds.get(entry.getId()); + newEntry.setChanges(changes); + addedEntryIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } else if (removedIds.contains(entry.getId())) { + entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now())); + } else if (addedEntryIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); + var newEntry = addedEntryIds.get(entry.getId()); + newEntry.setChanges(changes); + addedEntryIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } + }); + + newEntityLogEntries.removeAll(toRemove); + + addedEntryIds.forEach((k, v) -> { + if (v.getChanges().isEmpty()) { + v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); + } + newEntityLogEntries.add(v); + }); + + currentEntityLog.setEntityLogEntry(newEntityLogEntries); + + log.debug("Change computation took: {}", System.currentTimeMillis() - start); + return new EntityLogChanges(currentEntityLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty()); + } + + + private static Set getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(EntityLog currentEntityLog, List previouslyExistingEntries) { + + Set removed = new HashSet<>(previouslyExistingEntries); + currentEntityLog.getEntityLogEntry().forEach(removed::remove); + + Set removedIds = removed.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); + return removedIds; + } + + + private static Map getEntriesThatExistInCurrentButNotInPreviousEntityLog(EntityLog currentEntityLog, List previouslyExistingEntries) { + + Set currentExistingEntries = currentEntityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()) + .collect(Collectors.toSet()); + + previouslyExistingEntries.forEach(currentExistingEntries::remove); + Map addedIds = new HashMap<>(); + currentExistingEntries.forEach(entry -> { + addedIds.put(entry.getId(), entry); + }); + return addedIds; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 388ccba8..bf654a78 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -16,7 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; @@ -188,13 +188,13 @@ public class EntityLogCreatorService { } - private EntryState buildEntryState(Entity entity) { + private EntryState buildEntryState(IEntity IEntity) { - if (entity.applied()) { + if (IEntity.applied()) { return EntryState.APPLIED; - } else if (entity.skipped()) { + } else if (IEntity.skipped()) { return EntryState.SKIPPED; - } else if (entity.ignored()) { + } else if (IEntity.ignored()) { return EntryState.IGNORED; } else { return EntryState.REMOVED; @@ -202,13 +202,13 @@ public class EntityLogCreatorService { } - private EntryType buildEntryType(Entity entity, boolean isHint) { + private EntryType buildEntryType(IEntity IEntity, boolean isHint) { - if (entity instanceof TextEntity textEntity) { + if (IEntity instanceof TextEntity textEntity) { return getEntryType(isHint, textEntity.getEntityType()); - } else if (entity instanceof ManualEntity manualEntity) { + } else if (IEntity instanceof ManualEntity manualEntity) { return getEntryType(isHint, manualEntity.getEntityType()); - } else if (entity instanceof Image) { + } else if (IEntity instanceof Image) { return EntryType.IMAGE; } throw new UnsupportedOperationException("Entity subclass %s not implemented!"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index 15cb2a18..e89b7489 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -7,7 +7,7 @@ import java.util.NoSuchElementException; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; @@ -24,14 +24,14 @@ public class ManualChangesApplicationService { private final EntityCreationService entityCreationService; - public void recategorize(Entity entityToBeReCategorized, ManualRecategorization manualRecategorization) { + public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) { - if (entityToBeReCategorized instanceof Image image) { + if (IEntityToBeReCategorized instanceof Image image) { image.setImageType(ImageType.fromString(manualRecategorization.getType())); return; } // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed! - if (entityToBeReCategorized instanceof TextEntity textEntity) { + if (IEntityToBeReCategorized instanceof TextEntity textEntity) { TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode()); recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage()); recategorizedEntity.getManualOverwrite().addChange(manualRecategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java index 5d3be66b..56efb290 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java @@ -1,5 +1,131 @@ package com.iqser.red.service.redaction.v1.server.service.document; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.kie.api.runtime.KieSession; + +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.utils.DateConverter; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ComponentCreationService { + KieSession kieSession; + Set referencedEntities = new HashSet<>(); + + + public void firstOrElse(String ruleIdentifier, String category, Collection entities, String fallback) { + + String transformation = String.format("First found value or else '%s'", fallback); + String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback); + createComponent(ruleIdentifier, category, value, transformation, entities); + } + + + public void joining(String ruleIdentifier, String category, Collection entities) { + + joining(ruleIdentifier, category, entities, ", "); + } + + + public void joining(String ruleIdentifier, String category, Collection entities, String delimiter) { + + String transformation = String.format("Joining all values with '%s'", delimiter); + String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter)); + createComponent(ruleIdentifier, category, value, transformation, entities); + } + + + public void joiningUnique(String ruleIdentifier, String category, Collection entities) { + + joining(ruleIdentifier, category, entities, ", "); + } + + + public void joiningUnique(String ruleIdentifier, String category, Collection entities, String delimiter) { + + String transformation = String.format("Joining all values with '%s'", delimiter); + String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter)); + createComponent(ruleIdentifier, category, value, transformation, entities); + } + + + public void convertDates(String ruleIdentifier, String category, Collection entities) { + + convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy"); + } + + + public void convertDates(String ruleIdentifier, String category, Collection entities, String resultFormat) { + + String transformation = "Convert values of type to dd/MM/yyyy joined with ', '"; + String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", ")); + createComponent(ruleIdentifier, category, date, transformation, entities); + } + + + public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection entities) { + + entities.forEach(entity -> createComponent(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity))); + } + + + public void createComponent(String ruleIdentifier, String category, String value, String transformation, Collection references) { + + referencedEntities.addAll(references); + + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .category(category) + .value(value) + .transformation(transformation) + .references(new LinkedList<>(references))); + } + + + public void createComponent(String ruleIdentifier, String category, String value, String transformation, Entity reference) { + + referencedEntities.add(reference); + List referenceList = new LinkedList<>(); + referenceList.add(reference); + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .category(category) + .value(value) + .transformation(transformation) + .references(referenceList) + .build()); + } + + + public void createComponent(String ruleIdentifier, String category, String value, String transformation) { + + createComponent(ruleIdentifier, category, value, transformation, Collections.emptyList()); + } + + + public void createComponent(String ruleIdentifier, String category, String value) { + + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .category(category) + .value(value) + .transformation("") + .references(Collections.emptyList()) + .build()); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java new file mode 100644 index 00000000..7364bcdd --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import java.util.Comparator; + +import com.iqser.red.service.redaction.v1.server.model.component.Entity; + +public abstract class EntityComparators implements Comparator { + + private static class LongestEntity implements Comparator { + + @Override + public int compare(Entity Entity, Entity otherEntity) { + + return Integer.compare(Entity.getLength(), otherEntity.getLength()); + } + + } + + private static class FirstEntity implements Comparator { + + @Override + public int compare(Entity Entity, Entity otherEntity) { + + return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset()); + } + + } + + + public static Comparator length() { + + return new LongestEntity(); + } + + + public static Comparator start() { + + return new FirstEntity(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java similarity index 93% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/SectionFinderService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index 12299d51..3eeab748 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.service; +package com.iqser.red.service.redaction.v1.server.service.document; import java.util.Collection; import java.util.Collections; @@ -8,9 +8,11 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; @@ -19,13 +21,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -34,18 +34,18 @@ import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j -@Component +@Service @RequiredArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -class SectionFinderService { +public class SectionFinderService { @Timed("redactmanager_findSectionsToReanalyse") - public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Document document, AnalyzeRequest analyzeRequest) { + public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) { long start = System.currentTimeMillis(); Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set sectionsToReanalyse = new HashSet<>(); - for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { + for (EntityLogEntry entry : entityLog.getEntityLogEntry()) { if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { sectionsToReanalyse.add(entry.getSectionNumber()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java index 34a8660b..ee661e9e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java @@ -1,16 +1,26 @@ package com.iqser.red.service.redaction.v1.server.service.drools; -import java.util.Collections; +import java.util.LinkedList; import java.util.List; -import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.kie.api.runtime.KieContainer; +import org.kie.api.runtime.KieSession; +import org.kie.api.runtime.rule.QueryResults; +import org.kie.api.runtime.rule.QueryResultsRow; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Component; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -23,9 +33,61 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class ComponentDroolsExecutionService { - public List executeRules(Set entities, List allFileAttributes, List notFoundManualRedactionEntries, List list) { + RedactionServiceSettings settings; - return Collections.emptyList(); + + public List executeRules(KieContainer kieContainer, EntityLog entityLog, List fileAttributes) { + + KieSession kieSession = kieContainer.newKieSession(); + ComponentCreationService componentCreationService = new ComponentCreationService(kieSession); + + kieSession.setGlobal("componentCreationService", componentCreationService); + entityLog.getEntityLogEntry().stream().map(Entity::fromEntityLogEntry).forEach(kieSession::insert); + fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert); + + CompletableFuture completableFuture = CompletableFuture.supplyAsync(() -> { + kieSession.fireAllRules(); + return null; + }); + + try { + completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get(); + } catch (ExecutionException e) { + kieSession.dispose(); + if (e.getCause() instanceof TimeoutException) { + throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT); + } + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + List resultingFileAttributes = getFileAttributes(kieSession); + List components = getComponents(kieSession); + kieSession.dispose(); + return components; + } + + + public List getFileAttributes(KieSession kieSession) { + + List fileAttributes = new LinkedList<>(); + QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes"); + for (QueryResultsRow resultsRow : entitiesResult) { + fileAttributes.add((FileAttribute) resultsRow.get("$fileAttribute")); + } + return fileAttributes; + } + + + public List getComponents(KieSession kieSession) { + + List components = new LinkedList<>(); + QueryResults entitiesResult = kieSession.getQueryResults("getComponents"); + for (QueryResultsRow resultsRow : entitiesResult) { + components.add((Component) resultsRow.get("$component")); + } + return components; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index b059656b..8f4ddbd1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -47,6 +47,16 @@ public class RedactionStorageService { } + @SneakyThrows + public File getStoredObjectFile(String storageId) { + + File tempFile = File.createTempFile("temp", "data"); + tempFile.deleteOnExit(); + storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile); + return tempFile; + } + + @SneakyThrows public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java new file mode 100644 index 00000000..99847d07 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java @@ -0,0 +1,49 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Locale; + +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@UtilityClass +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DateConverter { + + static List formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH), + new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH), + new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH), + new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH), + new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH), + new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH)); + + + public String convertDate(String dateAsString, String resultFormat) { + + DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH); + Date date = null; + for (SimpleDateFormat format : formats) { + + try { + date = format.parse(dateAsString); + break; + } catch (Exception e) { + log.warn("Failed to parse date from string {}. \n{}", dateAsString, e.getMessage()); + // ignore, try next... + } + } + + if (date == null) { + return dateAsString; + } + + return resultDateFormat.format(date); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index ee409f37..cd0b91c1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -43,6 +43,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext; public class DocumineFloraTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/documine_flora.drl"); + private static final String COMPONENT_RULES = loadFromClassPath("drools/documine_flora_components.drl"); @Test @@ -148,7 +149,8 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); - doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES)); loadTypeForTest(); loadNerForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java index 8523bb52..98875593 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java @@ -13,7 +13,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.pdfbox.Loader; -import org.apache.pdfbox.io.MemoryUsageSetting; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.PDPageContentStream; @@ -36,10 +35,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle; -import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; -import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException; import lombok.RequiredArgsConstructor; @@ -67,14 +66,14 @@ public class AnnotationService { */ public AnnotateResponse annotate(AnnotateRequest annotateRequest) { - var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), + var storedObjectFile = redactionStorageService.getStoredObjectFile(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN)); var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId()); var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId()); - try (PDDocument pdDocument = Loader.loadPDF(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) { + try (PDDocument pdDocument = Loader.loadPDF(storedObjectFile)) { pdDocument.setAllSecurityToBeRemoved(true); dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java similarity index 99% rename from redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java index 5230a079..38a52644 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java @@ -32,7 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; -public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest { +public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 9e53f0fa..be3f9132 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -255,7 +255,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration var tmpFileName = "/tmp/" + filename.split("/")[2] + "_ENTITY_BBOX.pdf"; var fileResource = new ClassPathResource(filename + ".pdf"); - try (var fileStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(fileStream)) { + try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) { for (Page page : document.getPages()) { List entityPositionsOnPage = page.getEntities() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java index e6b17732..024660e8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java @@ -79,7 +79,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati var tmpFileName = "/tmp/" + filename.split("/")[2] + "_SEMANTIC_NODES_BBOX.pdf"; var fileResource = new ClassPathResource(filename + ".pdf"); - try (var fileStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(fileStream)) { + try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) { PdfVisualisationUtility.drawDocumentGraph(pdDocument, document); PdfVisualisationUtility.drawTextBlock(pdDocument, diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java index 009c3cc3..d25f4717 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.MockitoAnnotations; import org.springframework.boot.test.mock.mockito.MockBean; @@ -154,4 +155,12 @@ class DroolsSyntaxValidationServiceTest { assertFalse(droolsSyntaxValidation.isCompiled()); } + + @Test + @SneakyThrows + @Disabled + void fixAllRuleFiles() { + + } + } \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index b8422bd4..c5ae5d41 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -71,7 +71,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.textRange().start() < entity.textRange().end())); ClassPathResource resource = new ClassPathResource(filePath); - try (PDDocument pdDocument = Loader.loadPDF(resource.getInputStream())) { + try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) { Stream unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) .getNerEntityList() @@ -120,7 +120,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getTextRange().start() < entity.getTextRange().end())); ClassPathResource resource = new ClassPathResource(filePath); - try (PDDocument pdDocument = Loader.loadPDF(resource.getInputStream())) { + try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) { List validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) .getNerEntityList() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java index 30b6d20d..ef6cc46d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java @@ -21,6 +21,7 @@ public class LayoutParsingRequestProvider { var positionBlockFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION); var pageFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES); var simplifiedTextStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SIMPLIFIED_TEXT); + var viewerDocumentStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.VIEWER_DOCUMENT); return LayoutParsingRequest.builder() .layoutParsingType(layoutParsingType) .originFileStorageId(originFileStorageId) @@ -31,7 +32,7 @@ public class LayoutParsingRequestProvider { .positionBlockFileStorageId(positionBlockFileStorageId) .pageFileStorageId(pageFileStorageId) .sectionGridStorageId(sectionGridStorageId) - .simplifiedTextStorageId(simplifiedTextStorageId) + .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl new file mode 100644 index 00000000..a2470366 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl @@ -0,0 +1,558 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; + + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; + +global ComponentCreationService componentCreationService + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +query "getComponents" + $component: Component() + end + +//------------------------------------ Default Components rules ------------------------------------ + + +rule "StudyTitle.0.0: Study Title" + when + $titleCandidates: List() from collect (Entity(type == "study_title")) + then + componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!"); + end + + +rule "PerformingLaboratory.0.0: Performing Laboratory" + when + $laboratoryName: Entity(type == "laboratory_name") + $laboratoryCountry: Entity(type == "laboratory_country", $distance: Math.abs($laboratoryName.startOffset - startOffset)) + not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < $distance) + eval($distance < 80) + then + componentCreationService.createComponent("PerformingLaboratory.0.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!"); + end + +rule "PerformingLaboratory.0.1: Performing Laboratory" + when + $laboratoryName: Entity(type == "laboratory_name") + not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80) + then + componentCreationService.createComponent("DefaultComponents.0.1", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!"); + end + +rule "PerformingLaboratory.0.2: Performing Laboratory" + salience -1 + when + not Component(category == "Performing_Laboratory") + then + componentCreationService.createComponent("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback"); + end + + +rule "ReportNumber.0.0: Report number" + when + $reportNumberCandidates: List() from collect (Entity(type == "report_number")) + then + componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "No report number found!"); + end + + +rule "GLPStudy.0.0: GLP Study" + when + $glpStudy: Entity(type() == "glp_study") + then + componentCreationService.createComponent("GLPStudy.0.0", "GLP_study", "Yes", "Yes if present, No if not", $glpStudy); + end + +rule "GLPStudy.1.0: GLP Study" + when + not Entity(type() == "glp_study") + then + componentCreationService.createComponent("GLPStudy.1.0", "GLP_study", "No", "Yes if present, No if not"); + end + +rule "GLPStudy.2.0: GLP Study" + when + $first: Component(category == "GLP_study") + $second: Component(category == "GLP_study", value == $first.value, this != $first) + then + $first.addReferences($second.getReferences()); + retract($second); + end + + +rule "DefaultComponents.4.0: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number") + not Entity(type == "oecd_guideline_year") + then + componentCreationService.createComponent("DefaultComponents.4.0", "Test_Guidelines_1", $guidelineNumber.getValue(), "Only OECD Number found!"); + end + +rule "DefaultComponents.4.1: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2015") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "404") + then + componentCreationService.createComponent( + "DefaultComponents.4.1", + "Test_Guidelines_1", + "Nº 404: Acute Dermal Irritation/Corrosion (28/07/2015)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.2: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1981") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "403") + then + componentCreationService.createComponent( + "DefaultComponents.4.2", + "Test_Guidelines_1", + "Nº 403: Acute Inhalation Toxicity (12/05/1981)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.3: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "442B") + then + componentCreationService.createComponent( + "DefaultComponents.4.3", + "Test_Guidelines_1", + "Nº 442B: Skin Sensitization (27/06/2018)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.4: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2016") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "487") + then + componentCreationService.createComponent( + "DefaultComponents.4.4", + "Test_Guidelines_1", + "Nº 487: Micronucleus Human Lymphocytes (2016)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.5: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2004") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "428") + then + componentCreationService.createComponent( + "DefaultComponents.4.5", + "Test_Guidelines_1", + "Nº 428: Split-Thickness Skin test (2004)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.6: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "442A") + then + componentCreationService.createComponent( + "DefaultComponents.4.6", + "Test_Guidelines_1", + "Nº 442A: Skin Sensitization (23/07/2018)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.7: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "404") + then + componentCreationService.createComponent( + "DefaultComponents.4.7", + "Test_Guidelines_1", + "Nº 404: Acute Dermal Irritation/Corrosion (24/04/2002)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.8: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1992") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "406") + then + componentCreationService.createComponent( + "DefaultComponents.4.8", + "Test_Guidelines_1", + "Nº 406: Skin Sensitisation (1992)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.9: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "438") + then + componentCreationService.createComponent( + "DefaultComponents.4.9", + "Test_Guidelines_1", + "Nº 438: Eye Irritation (26/06/2018)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.10: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2009") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "436") + then + componentCreationService.createComponent( + "DefaultComponents.4.10", + "Test_Guidelines_1", + "Nº 436: Acute Inhalation Toxicity – Acute Toxic Class Method (08/09/2009)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.11: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "429") + then + componentCreationService.createComponent( + "DefaultComponents.4.11", + "Test_Guidelines_1", + "Nº 429: Skin Sensitisation: Local Lymph Node Assay (24/04/2002)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.12: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2020") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "471") + then + componentCreationService.createComponent( + "DefaultComponents.4.12", + "Test_Guidelines_1", + "Nº 471: Bacterial Reverse Mutation Test (26/06/2020)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.13: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1987") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "405") + then + componentCreationService.createComponent( + "DefaultComponents.4.13", + "Test_Guidelines_1", + "Nº 405: Acute Eye Irritation/Corrosion (24/02/1987)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.14: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1997") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "471") + then + componentCreationService.createComponent( + "DefaultComponents.4.14", + "Test_Guidelines_1", + "Nº 471: Bacterial Reverse Mutation Test (21/07/1997)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.15: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2009") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "403") + then + componentCreationService.createComponent( + "DefaultComponents.4.15", + "Test_Guidelines_1", + "Nº 403: Acute Inhalation Toxicity (08/09/2009)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.16: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2018") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "433") + then + componentCreationService.createComponent( + "DefaultComponents.4.16", + "Test_Guidelines_1", + "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (27/06/2018)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.17: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2019") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "439") + then + componentCreationService.createComponent( + "DefaultComponents.4.17", + "Test_Guidelines_1", + "Nº 439: Skin Irritation (2019)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.18: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2001") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "425") + then + componentCreationService.createComponent( + "DefaultComponents.4.18", + "Test_Guidelines_1", + "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (17/12/2001)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.19: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "402") + then + componentCreationService.createComponent( + "DefaultComponents.4.19", + "Test_Guidelines_1", + "Nº 402: Acute Dermal Toxicity (09/10/2017)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.20: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1981") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "404") + then + componentCreationService.createComponent( + "DefaultComponents.4.20", + "Test_Guidelines_1", + "Nº 404: Acute Dermal Irritation/Corrosion (12/05/1981)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.21: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2016") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "474") + then + componentCreationService.createComponent( + "DefaultComponents.4.21", + "Test_Guidelines_1", + "Nº 474: Micronucleus Bone Marrow Cells Rat (2016)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.22: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "433") + then + componentCreationService.createComponent( + "DefaultComponents.4.22", + "Test_Guidelines_1", + "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (09/10/2017)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.23: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1992") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "404") + then + componentCreationService.createComponent( + "DefaultComponents.4.23", + "Test_Guidelines_1", + "Nº 404: Acute Dermal Irritation/Corrosion (17/07/1992)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.24: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2010") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "429") + then + componentCreationService.createComponent( + "DefaultComponents.4.24", + "Test_Guidelines_1", + "Nº 429: Skin Sensitisation (23/07/2010)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.25: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2017") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "405") + then + componentCreationService.createComponent( + "DefaultComponents.4.25", + "Test_Guidelines_1", + "Nº 405: Acute Eye Irritation/Corrosion (09/10/2017)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.26: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "1987") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "402") + then + componentCreationService.createComponent( + "DefaultComponents.4.26", + "Test_Guidelines_1", + "Nº 402: Acute Dermal Toxicity (24/02/1987)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.27: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2012") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "405") + then + componentCreationService.createComponent( + "DefaultComponents.4.27", + "Test_Guidelines_1", + "Nº 405: Acute Eye Irritation/Corrosion (02/10/2012)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.28: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2002") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "405") + then + componentCreationService.createComponent( + "DefaultComponents.4.28", + "Test_Guidelines_1", + "Nº 405: Acute Eye Irritation/Corrosion (24/04/2002)", + "OECD Number and guideline year mapped!" + ); + end + +rule "DefaultComponents.4.29: Test Guideline 1" + when + $guidelineNumber: Entity(type == "oecd_guideline_number", value == "2008") + $guidelineYear: Entity(type == "oecd_guideline_year", value == "425") + then + componentCreationService.createComponent( + "DefaultComponents.4.29", + "Test_Guidelines_1", + "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (03/10/2008)", + "OECD Number and guideline year mapped!" + ); + end + + +rule "DefaultComponents.5.0: Test Guideline 2" + when + $epaGuideLines: List() from collect (Entity(type() == "epa_guideline")) + $ecGuideLines: List() from collect (Entity(type() == "ec_guideline")) + then + componentCreationService.joining("DefaultComponents.5.0", + "Test_Guideline_2", + Stream.of( + $epaGuideLines.stream(), + $ecGuideLines.stream()) + .flatMap(a -> a) + .toList() + ); + end + + +rule "DefaultComponents.6.0: Experimental Starting Date" + when + $startDates: List(!isEmpty()) from collect (Entity(type() == "experimental_start_date")) + then + componentCreationService.convertDates("DefaultComponents.6.0", "Experimental_Starting_Date", $startDates); + end + + +rule "DefaultComponents.7.0: Experimental Completion Date" + when + $endDates: List(!isEmpty()) from collect (Entity(type() == "experimental_end_date")) + then + componentCreationService.convertDates("DefaultComponents.7.0", "Experimental_Completion_Date", $endDates); + end + + +rule "DefaultComponents.8.0: Certificate of analysis batch identification" + when + $batchNumbers: List(!isEmpty()) from collect (Entity(type() == "batch_number")) + then + componentCreationService.joiningUnique("DefaultComponents.8.0", "Batch_Number", $batchNumbers); + end + + +rule "DefaultComponents.999.0: Create components for all unmapped entities." + salience -999 + when + $allEntities: List(!isEmpty()) from collect (Entity() + then + componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities); + end + + + +//------------------------------------ Component merging rules ------------------------------------ + +rule "X.0.0: merge duplicate component references" + when + $first: Component() + $duplicate: Component(this != $first, category == category, value == value) + then + $first.getReferences().addAll($duplicate.getReferences()); + retract($duplicate); + end + + diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 895d9f5b..b8709285 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -58,6 +58,7 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary + //------------------------------------ queries ------------------------------------ query "getFileAttributes" diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 5183c0b6..198f38cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -58,6 +58,7 @@ global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary + // --------------------------------------- queries ------------------------------------------------------------------- query "getFileAttributes" diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index 2dbe211c..1a57e91b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -47,7 +47,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType global Document document global EntityCreationService entityCreationService