diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java index a3f47de9..11c9eecd 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.model; +import java.util.LinkedList; import java.util.List; import lombok.AccessLevel; @@ -16,6 +17,18 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class DroolsSyntaxValidation { - boolean compiled; - List droolsSyntaxErrorMessages; + @Builder.Default + List droolsSyntaxErrorMessages = new LinkedList<>(); + + + public void addErrorMessage(int line, int column, String message) { + + getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build()); + } + + public boolean isCompiled() { + + return droolsSyntaxErrorMessages.isEmpty(); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 17f32180..9d377f6e 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -12,11 +12,11 @@ plugins { description = "redaction-service-server-v1" -val layoutParserVersion = "0.25.0" +val layoutParserVersion = "0.70.0" val jacksonVersion = "2.15.2" val droolsVersion = "8.43.0.Final" -val pdfBoxVersion = "3.0.0-alpha2" -val persistenceServiceVersion = "2.160.0" +val pdfBoxVersion = "3.0.0" +val persistenceServiceVersion = "2.168.0" configurations { all { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java index 5d904706..ab3bd66f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java @@ -12,7 +12,6 @@ import org.springframework.context.annotation.Import; import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java similarity index 92% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index 02f1d8fa..2cc7f8f4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.settings; +package com.iqser.red.service.redaction.v1.server; import org.springframework.boot.context.properties.ConfigurationProperties; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/ControllerAdvice.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/ControllerAdvice.java index c92c6395..95e659b1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/ControllerAdvice.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/ControllerAdvice.java @@ -1,8 +1,8 @@ package com.iqser.red.service.redaction.v1.server.controller; import com.iqser.red.commons.spring.ErrorMessage; -import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; -import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; +import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index d33d5b52..bd9c9951 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -1,12 +1,13 @@ package com.iqser.red.service.redaction.v1.server.controller; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; import com.iqser.red.service.redaction.v1.model.RuleValidationModel; import com.iqser.red.service.redaction.v1.resources.RedactionResource; -import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; -import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -16,14 +17,14 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class RedactionController implements RedactionResource { - private final DroolsExecutionService droolsExecutionService; + private final DroolsSyntaxValidationService droolsSyntaxValidationService; @Override - public DroolsSyntaxValidation testRules(RuleValidationModel rulesValidationModel) { + public DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel) { try { - return droolsExecutionService.testRules(rulesValidationModel.getRulesString()); + return droolsSyntaxValidationService.testRules(rulesValidationModel); } catch (Exception e) { throw new RulesValidationException("Could not test rules: " + e.getMessage(), e); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java index 1a800e0b..47c637fa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java @@ -1,10 +1,11 @@ package com.iqser.red.service.redaction.v1.server.controller; +import java.util.Collections; + import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource; -import com.iqser.red.service.redaction.v1.server.redaction.service.RuleBuilderModelService; import lombok.RequiredArgsConstructor; @@ -12,13 +13,16 @@ import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public class RuleBuilderController implements RuleBuilderResource { - private final RuleBuilderModelService ruleBuilderModelService; - @Override public RuleBuilderModel getRuleBuilderModel() { - return ruleBuilderModelService.getRuleBuilderModel(); + RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); + + ruleBuilderModel.setWhenClauses(Collections.emptyList()); + ruleBuilderModel.setThenConditions(Collections.emptyList()); + + return ruleBuilderModel; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/RuleIdentifier.java deleted file mode 100644 index 776a50a9..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/RuleIdentifier.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.entity; - -import java.util.Objects; - -public record RuleIdentifier(String type, Integer unit, Integer id) { - - public static RuleIdentifier fromString(String identifier) { - - String[] values = identifier.split("\\."); - if (values.length != 3) { - throw new IllegalArgumentException("Illegal rule identifier provided: " + identifier); - } - String type = values[0]; - Integer group = Integer.parseInt(values[1]); - Integer id = Integer.parseInt(values[2]); - return new RuleIdentifier(type, group, id); - } - - - public static RuleIdentifier empty() { - - return new RuleIdentifier("", null, null); - } - - - @Override - public String toString() { - - StringBuilder sb = new StringBuilder(); - sb.append(type()); - if (Objects.nonNull(unit()) && Objects.nonNull(id())) { - sb.append(".").append(unit()).append(".").append(id()); - } else if (Objects.nonNull(id())) { - sb.append(".*.").append(id()); - } else if (Objects.nonNull(unit())) { - sb.append(".").append(unit()).append(".*"); - } - return sb.toString(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/DroolsTimeoutException.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/DroolsTimeoutException.java deleted file mode 100644 index 57b9f9c2..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/DroolsTimeoutException.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.exception; - -import lombok.Data; - -@Data -public class DroolsTimeoutException extends RuntimeException { - - private static final String DROOLS_TIMEOUT_MESSAGE = "Timeout during rules execution. Possible endless loop?"; - - private boolean reported; - - public DroolsTimeoutException(Throwable cause, boolean reported) { - - super(DROOLS_TIMEOUT_MESSAGE, cause); - this.reported = reported; - } - - public DroolsTimeoutException(boolean reported) { - - super(DROOLS_TIMEOUT_MESSAGE); - this.reported = reported; - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java new file mode 100644 index 00000000..1cacd5b1 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.server.model; + +import org.kie.api.runtime.KieContainer; + +public record KieWrapper(KieContainer container, long rulesVersion) { + + public static KieWrapper empty() { + + return new KieWrapper(null, -1); + } + + + public boolean isPresent() { + + return container != null && rulesVersion >= 0; + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ManualEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java similarity index 82% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ManualEntity.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java index cffe0394..e3104457 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/ManualEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java @@ -1,14 +1,15 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; +package com.iqser.red.service.redaction.v1.server.model; import java.util.List; import java.util.PriorityQueue; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.Entity; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ManualChangeOverwrite; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -20,9 +21,9 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class ManualEntity implements Entity { +public class ManualEntity implements IEntity { - // must be mapped into a TextEntity as is for comments to work correctly + // The id must be mapped into a TextEntity as is for comments to work correctly String id; String value; List entityPosition; @@ -88,4 +89,18 @@ public class ManualEntity implements Entity { .build(); } + + @Override + public TextRange getTextRange() { + + return new TextRange(-1, -1); + } + + + @Override + public String type() { + + return getManualOverwrite().getType().orElse(type); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java similarity index 86% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntities.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java index a8a3b4d2..71c1c56f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java @@ -1,10 +1,10 @@ -package com.iqser.red.service.redaction.v1.server.redaction.adapter; +package com.iqser.red.service.redaction.v1.server.model; import java.util.LinkedList; import java.util.List; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RectangleWithPage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java similarity index 94% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RectangleWithPage.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java index a27c23a2..58879ce8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RectangleWithPage.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; +package com.iqser.red.service.redaction.v1.server.model; import java.awt.geom.Rectangle2D; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java new file mode 100644 index 00000000..d820e712 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.redaction.v1.server.model.component; + +import java.util.Collection; +import java.util.List; + +import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class Component { + + RuleIdentifier matchedRule; + String category; + String value; + String transformation; + + List references; + + + public boolean addReference(Entity entity) { + + return references.add(entity); + } + + + public boolean addReferences(Collection entities) { + + return references.addAll(entities); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java new file mode 100644 index 00000000..724812ca --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java @@ -0,0 +1,101 @@ +package com.iqser.red.service.redaction.v1.server.model.component; + +import java.util.List; +import java.util.Set; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.experimental.FieldDefaults; + +@Getter +@Builder +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class Entity { + + String id; + String type; + EntryType entryType; + EntryState state; + String value; + String reason; + String matchedRule; + String legalBasis; + + boolean imported; + + String section; + float[] color; + + List positions; + int sectionNumber; + + String textBefore; + String textAfter; + + int startOffset; + int endOffset; + int length; + + boolean imageHasTransparency; + + boolean isDictionaryEntry; + boolean isDossierDictionaryEntry; + + boolean excluded; + + List changes; + + List manualChanges; + + Set engines; + + Set reference; + + Set importedRedactionIntersections; + + + public static Entity fromEntityLogEntry(EntityLogEntry e) { + + return Entity.builder() + .id(e.getId()) + .type(e.getType()) + .entryType(e.getEntryType()) + .state(e.getState()) + .value(e.getValue()) + .reason(e.getReason()) + .matchedRule(e.getMatchedRule()) + .legalBasis(e.getLegalBasis()) + .imported(e.isImported()) + .section(e.getSection()) + .color(e.getColor()) + .positions(e.getPositions()) + .sectionNumber(e.getSectionNumber()) + .textBefore(e.getTextBefore()) + .textAfter(e.getTextAfter()) + .startOffset(e.getStartOffset()) + .endOffset(e.getEndOffset()) + .length(e.getValue().length()) + .imageHasTransparency(e.isImageHasTransparency()) + .isDictionaryEntry(e.isDictionaryEntry()) + .isDossierDictionaryEntry(e.isDossierDictionaryEntry()) + .excluded(e.isExcluded()) + .changes(e.getChanges()) + .manualChanges(e.getManualChanges()) + .engines(e.getEngines()) + .reference(e.getReference()) + .importedRedactionIntersections(e.getImportedRedactionIntersections()) + .build(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java similarity index 92% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index 0bb75596..f935b9b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import static java.lang.String.format; @@ -15,10 +15,10 @@ import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; -import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; +import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.utils.Patterns; import lombok.Data; import lombok.Getter; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryEntries.java similarity index 88% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryEntries.java index ac80349b..c921ccea 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryEntries.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import java.util.HashSet; import java.util.Set; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrement.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIncrement.java similarity index 75% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrement.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIncrement.java index 961636cb..a240105b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrement.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIncrement.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import java.util.Set; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrementValue.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIncrementValue.java similarity index 70% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrementValue.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIncrementValue.java index c50ed785..0202d038 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrementValue.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryIncrementValue.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java similarity index 96% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java index da25189e..13284967 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import java.io.Serializable; import java.util.HashMap; @@ -7,7 +7,7 @@ import java.util.stream.Collectors; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryRepresentation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryRepresentation.java similarity index 87% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryRepresentation.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryRepresentation.java index 7d084784..ee1a7521 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryRepresentation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryRepresentation.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import java.util.ArrayList; import java.util.HashMap; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryVersion.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryVersion.java similarity index 76% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryVersion.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryVersion.java index e1f46683..a5a21cf2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryVersion.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryVersion.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/SearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/SearchImplementation.java similarity index 96% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/SearchImplementation.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/SearchImplementation.java index bd095b02..6d8ead3e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/SearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/SearchImplementation.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import java.util.ArrayList; import java.util.Collection; @@ -9,7 +9,7 @@ import java.util.stream.Collectors; import org.ahocorasick.trie.Trie; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import lombok.Data; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/TenantDictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TenantDictionary.java similarity index 82% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/TenantDictionary.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TenantDictionary.java index 7b666eb1..af964cc9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/TenantDictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/TenantDictionary.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; +package com.iqser.red.service.redaction.v1.server.model.dictionary; import java.util.HashMap; import java.util.Map; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ConsecutiveBoundaryCollector.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/ConsecutiveBoundaryCollector.java similarity index 96% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ConsecutiveBoundaryCollector.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/ConsecutiveBoundaryCollector.java index a77430a6..4986b3f3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ConsecutiveBoundaryCollector.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/ConsecutiveBoundaryCollector.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; +package com.iqser.red.service.redaction.v1.server.model.document; import java.util.LinkedList; import java.util.List; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentData.java similarity index 92% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentData.java index 11933064..7575203e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentData.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.data; +package com.iqser.red.service.redaction.v1.server.model.document; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTree.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java similarity index 74% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTree.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java index 91432036..64077a54 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTree.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java @@ -1,20 +1,21 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; +package com.iqser.red.service.redaction.v1.server.model.document; import static java.lang.String.format; import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.GenericSemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.GenericSemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -135,6 +136,48 @@ public class DocumentTree { } + public Optional getNextSibling(List treeId) { + + var siblingTreeId = getNextSiblingId(treeId); + if (!entryExists(siblingTreeId)) { + return Optional.empty(); + } + return Optional.of(getEntryById(siblingTreeId).getNode()); + } + + + public List getNextSiblingId(List treeId) { + + List siblingTreeId = new LinkedList<>(); + for (int i = 0; i < treeId.size() - 1; i++) { + siblingTreeId.add(treeId.get(i)); + } + siblingTreeId.add(treeId.get(treeId.size() - 1) + 1); + return siblingTreeId; + } + + + public Optional getPreviousSibling(List treeId) { + + var siblingTreeId = getPreviousSiblingId(treeId); + if (!entryExists(siblingTreeId)) { + return Optional.empty(); + } + return Optional.of(getEntryById(siblingTreeId).getNode()); + } + + + public List getPreviousSiblingId(List treeId) { + + List siblingTreeId = new LinkedList<>(); + for (int i = 0; i < treeId.size() - 1; i++) { + siblingTreeId.add(treeId.get(i)); + } + siblingTreeId.add(treeId.get(treeId.size() - 1) - 1); + return siblingTreeId; + } + + public Entry getEntryById(List treeId) { if (treeId.isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextRange.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextRange.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java index 6b624e60..370d19d7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextRange.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; +package com.iqser.red.service.redaction.v1.server.model.document; import static java.lang.String.format; @@ -6,7 +6,7 @@ import java.util.Collection; import java.util.LinkedList; import java.util.List; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.EqualsAndHashCode; import lombok.Setter; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java similarity index 64% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java index 4db81e91..b86bc323 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.entity; +package com.iqser.red.service.redaction.v1.server.model.document.entity; public enum EntityType { ENTITY, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java similarity index 85% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/Entity.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java index d35b939e..9d43e342 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java @@ -1,13 +1,16 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.entity; +package com.iqser.red.service.redaction.v1.server.model.document.entity; import java.util.Collection; import java.util.HashSet; import java.util.PriorityQueue; import java.util.Set; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; + import lombok.NonNull; -public interface Entity { +public interface IEntity { PriorityQueue getMatchedRuleList(); @@ -15,7 +18,40 @@ public interface Entity { ManualChangeOverwrite getManualOverwrite(); - // Don't use default accessor pattern (e.g. isIgnored()), as it might lead to errors in drools due to property-specific optimization of the drools planner. + String getValue(); + + + TextRange getTextRange(); + + + String type(); + + + default int length() { + + return value().length(); + } + + + default String value() { + + return getManualOverwrite().getValue().orElse(getValue()); + } + + + // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner. + default boolean applied() { + + return getManualOverwrite().getApplied().orElse(getMatchedRule().isApplied()); + } + + + default boolean skipped() { + + return !applied(); + } + + default boolean ignored() { return getManualOverwrite().getIgnored().orElse(getMatchedRule().isIgnored()); @@ -34,9 +70,9 @@ public interface Entity { } - default boolean applied() { + default boolean active() { - return getManualOverwrite().getApplied().orElse(getMatchedRule().isApplied()); + return !(removed() || ignored()); } @@ -52,12 +88,6 @@ public interface Entity { } - default boolean active() { - - return !(removed() || ignored()); - } - - default void redact(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) { if (legalBasis.isBlank() || legalBasis.isEmpty()) { @@ -79,17 +109,6 @@ public interface Entity { } - default void force(@NonNull String ruleIdentifier, String reason, String legalBasis) { - - addMatchedRule(MatchedRule.builder() - .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) - .reason(reason) - .legalBasis(getLegalBasisOrPreviousLegalBasisOrPlaceHolder(legalBasis)) - .applied(true) - .build()); - } - - default void skip(@NonNull String ruleIdentifier, String reason) { addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build()); @@ -108,18 +127,6 @@ public interface Entity { } - private String getLegalBasisOrPreviousLegalBasisOrPlaceHolder(String legalBasis) { - - if (legalBasis == null || legalBasis.isBlank() || legalBasis.isEmpty()) { - if (getMatchedRule() == null || !getMatchedRule().isApplied()) { - return "n-a"; - } - return getMatchedRule().getLegalBasis(); - } - return legalBasis; - } - - default void applyWithLineBreaks(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) { if (legalBasis.isBlank() || legalBasis.isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ManualChangeOverwrite.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java similarity index 98% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ManualChangeOverwrite.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java index e28fda4c..a6b0ee3e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ManualChangeOverwrite.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.entity; +package com.iqser.red.service.redaction.v1.server.model.document.entity; import java.util.Collections; import java.util.Comparator; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java similarity index 82% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/MatchedRule.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java index 107debb6..fe5e4ad8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java @@ -1,10 +1,13 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.entity; +package com.iqser.red.service.redaction.v1.server.model.document.entity; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; + import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -19,12 +22,11 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public final class MatchedRule implements Comparable { - public static final String FINAL_TYPE = "FINAL"; - public static final String ELIMINATION_RULE_TYPE = "X"; - private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE); + public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL"); + public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X"); + private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE); - @Builder.Default - RuleIdentifier ruleIdentifier = RuleIdentifier.empty(); + RuleIdentifier ruleIdentifier; @Builder.Default String reason = ""; @Builder.Default @@ -40,7 +42,7 @@ public final class MatchedRule implements Comparable { public static MatchedRule empty() { - return MatchedRule.builder().build(); + return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/PositionOnPage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/PositionOnPage.java similarity index 83% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/PositionOnPage.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/PositionOnPage.java index 195f3774..36ea5970 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/PositionOnPage.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/PositionOnPage.java @@ -1,9 +1,9 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.entity; +package com.iqser.red.service.redaction.v1.server.model.document.entity; import java.awt.geom.Rectangle2D; import java.util.List; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/TextEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java similarity index 91% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/TextEntity.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index b2615bc9..1110f7b7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/TextEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.entity; +package com.iqser.red.service.redaction.v1.server.model.document.entity; import java.awt.geom.Rectangle2D; import java.util.Collection; @@ -10,11 +10,11 @@ import java.util.Map; import java.util.PriorityQueue; import java.util.Set; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.utils.IdBuilder; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -28,7 +28,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) @EqualsAndHashCode(onlyExplicitlyIncluded = true) -public class TextEntity implements Entity { +public class TextEntity implements IEntity { // primary key @EqualsAndHashCode.Include @@ -50,7 +50,6 @@ public class TextEntity implements Entity { @Builder.Default Set engines = new HashSet<>(); - // inferred on graph insertion String value; String textBefore; @@ -201,4 +200,11 @@ public class TextEntity implements Entity { return sb.toString(); } + + @Override + public String type() { + + return getManualOverwrite().getType().orElse(type); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java similarity index 89% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Document.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java index e1fd8481..4201f353 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.awt.geom.Rectangle2D; import java.util.Collections; @@ -10,10 +10,10 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Footer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java similarity index 83% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Footer.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java index f62e3d99..780b965d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Footer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java @@ -1,12 +1,12 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/GenericSemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/GenericSemanticNode.java similarity index 52% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/GenericSemanticNode.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/GenericSemanticNode.java index 3ed31fdc..e97efa3d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/GenericSemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/GenericSemanticNode.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; public interface GenericSemanticNode extends SemanticNode { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Header.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java similarity index 83% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Header.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java index 41b1b549..44c4c103 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Header.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java @@ -1,12 +1,12 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Headline.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java similarity index 85% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Headline.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java index b57e7873..5bdc020c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Headline.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java @@ -1,13 +1,13 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java similarity index 60% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Image.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java index b61cfc23..2f0c8a2e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java @@ -1,21 +1,23 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.awt.geom.Rectangle2D; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.PriorityQueue; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.Entity; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ManualChangeOverwrite; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -30,7 +32,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class Image implements GenericSemanticNode, Entity { +public class Image implements GenericSemanticNode, IEntity { List treeId; String id; @@ -77,6 +79,20 @@ public class Image implements GenericSemanticNode, Entity { } + @Override + public TextRange getTextRange() { + + return GenericSemanticNode.super.getTextRange(); + } + + + @Override + public String type() { + + return getManualOverwrite().getType().orElse(imageType.toString()); + } + + @Override public String toString() { @@ -92,4 +108,23 @@ public class Image implements GenericSemanticNode, Entity { return bBoxPerPage; } + + @Override + public String getValue() { + + return NodeType.IMAGE + ":" + camelCase(imageType.toString()); + } + + + private String camelCase(String name) { + + return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH); + } + + + public int length() { + + return 0; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java similarity index 88% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java index 5db33ec8..397d7b11 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.Locale; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java similarity index 75% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java index 135acb4e..8fe8a291 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.Locale; @@ -16,6 +16,6 @@ public enum NodeType { public String toString() { - return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT); + return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ENGLISH); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Page.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java similarity index 84% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Page.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java index 4ea63a53..18aee858 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Page.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java @@ -1,12 +1,12 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Paragraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java similarity index 80% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Paragraph.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java index 4fb062c5..6c9794bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Paragraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java @@ -1,12 +1,12 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java similarity index 86% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Section.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java index df76d026..749de494 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java @@ -1,13 +1,13 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java similarity index 98% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionIdentifier.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java index 12c3f4e9..2ff3a30f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.util.Collections; import java.util.LinkedList; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java similarity index 93% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java index 71b24bb5..2d8f4726 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import static java.lang.String.format; @@ -9,17 +9,18 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; public interface SemanticNode { @@ -170,6 +171,30 @@ public interface SemanticNode { } + /** + * Returns the next sibling node of this SemanticNode in the document tree, if any. + * If there is no next sibling node, an empty Optional is returned. + * + * @return Optional containing the next sibling node, or empty if there is none + */ + default Optional getNextSibling() { + + return getDocumentTree().getNextSibling(getTreeId()); + } + + + /** + * Returns the previous sibling node of this SemanticNode in the document tree, if any. + * If there is no previous sibling node, an empty Optional is returned. + * + * @return Optional containing the previous sibling node, or empty if there is none + */ + default Optional getPreviousSibling() { + + return getDocumentTree().getPreviousSibling(getTreeId()); + } + + /** * Leaf means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden. * Currently only Sections, Images, and Tables are not leaves. diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Table.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Table.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java index 6b9ac3de..a85acb28 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/Table.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import static java.lang.String.format; @@ -10,10 +10,10 @@ import java.util.Set; import java.util.stream.IntStream; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCell.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java similarity index 85% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCell.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java index 4775e110..6165c618 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCell.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.awt.geom.Rectangle2D; import java.util.HashMap; @@ -7,10 +7,10 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import lombok.AccessLevel; import lombok.AllArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java similarity index 95% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java index 4699ee08..49d707ed 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.textblock; +package com.iqser.red.service.redaction.v1.server.model.document.textblock; import static java.lang.String.format; @@ -12,10 +12,10 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java index 99089681..d180c5c9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.textblock; +package com.iqser.red.service.redaction.v1.server.model.document.textblock; import static java.lang.String.format; @@ -10,8 +10,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import lombok.AccessLevel; import lombok.Data; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java similarity index 92% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java index df0d4ceb..50e76203 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.textblock; +package com.iqser.red.service.redaction.v1.server.model.document.textblock; import static java.lang.String.format; @@ -10,9 +10,9 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; public interface TextBlock extends CharSequence { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlockCollector.java similarity index 94% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlockCollector.java index bb98abe3..964238b0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlockCollector.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.textblock; +package com.iqser.red.service.redaction.v1.server.model.document.textblock; import java.util.Set; import java.util.function.BiConsumer; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicQuery.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicQuery.java new file mode 100644 index 00000000..ca636ad3 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicQuery.java @@ -0,0 +1,15 @@ +package com.iqser.red.service.redaction.v1.server.model.drools; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicQuery { + + String name; + int line; + String code; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicRule.java new file mode 100644 index 00000000..d6f60188 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicRule.java @@ -0,0 +1,39 @@ +package com.iqser.red.service.redaction.v1.server.model.drools; + +import org.drools.drl.ast.descr.RuleDescr; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class BasicRule { + + @EqualsAndHashCode.Include + RuleIdentifier identifier; + String name; + String code; + int line; + + + public static BasicRule fromRuleDescr(RuleDescr rule, String rulesString) { + + RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName()); + String nameWithoutIdentifier = rule.getName().replace(identifier + ":", ""); + String code = rulesString.substring(rule.getStartCharacter(), rule.getEndCharacter()); + return new BasicRule(identifier, nameWithoutIdentifier, code, rule.getLine()); + } + + + @Override + public String toString() { + + return "BasicRule[identifier=" + identifier + ", name=" + name + ", code=" + code + ']'; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java new file mode 100644 index 00000000..5b75502a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java @@ -0,0 +1,15 @@ +package com.iqser.red.service.redaction.v1.server.model.drools; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +public record RuleClass(RuleType ruleType, List ruleUnits) { + + public Optional findRuleUnitByInteger(Integer unit) { + + return ruleUnits.stream() + .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java new file mode 100644 index 00000000..8dc3f863 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java @@ -0,0 +1,76 @@ +package com.iqser.red.service.redaction.v1.server.model.drools; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public final class RuleFileBluePrint { + + String imports; + int importLine; + String globals; + int globalsLine; + List queries; + List ruleClasses; + DroolsSyntaxValidation droolsSyntaxValidation; + + + public Optional findRuleClassByType(RuleType ruleType) { + + return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst(); + } + + + public List findRulesByIdentifier(RuleIdentifier ruleIdentifier) { + + if (Objects.isNull(ruleIdentifier.unit())) { + return findRuleClassByType(ruleIdentifier.type()).map(RuleClass::ruleUnits) + .orElse(Collections.emptyList()) + .stream() + .flatMap(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier))) + .toList(); + } + return findRuleClassByType(ruleIdentifier.type()).flatMap(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit())) + .map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier))) + .orElse(Stream.empty()) + .toList(); + } + + + public List getAllRuleIdentifiers() { + + return streamAllRules().map(BasicRule::getIdentifier).collect(Collectors.toList()); + } + + + public Stream streamAllRules() { + + return getRuleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream); + } + + + @Override + public String toString() { + + return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']'; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java new file mode 100644 index 00000000..3dbf0fb3 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java @@ -0,0 +1,92 @@ +package com.iqser.red.service.redaction.v1.server.model.drools; + +import java.util.Arrays; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import lombok.NonNull; + +public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) { + + public static RuleIdentifier fromName(String name) { + + String identifier = name.split(":")[0]; + return fromString(identifier); + } + + + public static RuleIdentifier fromString(String identifier) { + + String[] values = identifier.split("\\."); + if (values.length != 3) { + throw new IllegalArgumentException(String.format("Identifier %s must conform to the pattern \"[a-zA-Z0-9]+.\\d+.\\d+\"", identifier)); + } + RuleType type = RuleType.fromString(values[0]); + Integer group = Integer.parseInt(values[1]); + Integer id = Integer.parseInt(values[2]); + return new RuleIdentifier(type, group, id); + } + + + public static RuleIdentifier empty() { + + return new RuleIdentifier(new RuleType(""), null, null); + } + + + public static Set fromListOfIdentifiersString(String input) { + + return Arrays.stream(input.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet()); + } + + + private static Integer parseIntOrStar(String value) { + + return !value.equals("*") ? Integer.parseInt(value) : null; + } + + + /** + * This is used to filter rules, if the field Integer unit or Integer id is null, the field will match any other RuleIdentifier. + * Therefore, to compare RuleIdentifiers one should always use the function matches. + * + * @param ruleIdentifier RuleIdentifier to check if this one matches it. + * @return true, if all fields match. If a field is null, it matches any field. + */ + public boolean matches(RuleIdentifier ruleIdentifier) { + + return ruleIdentifier.type().equals(this.type()) && // + (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && // + (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id())); + + } + + + @Override + public String toString() { + + StringBuilder sb = new StringBuilder(); + sb.append(type().name()); + if (Objects.nonNull(unit())) { + sb.append("."); + sb.append(unit()); + } else { + sb.append(".*"); + } + if (Objects.nonNull(id())) { + sb.append("."); + sb.append(id()); + } else { + sb.append(".*"); + } + return sb.toString(); + } + + + public String toRuleUnitString() { + + return String.format("Rule unit: %s.%d", type.name(), unit); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleType.java new file mode 100644 index 00000000..2c8eec2e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleType.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.server.model.drools; + +import java.util.regex.Pattern; + + +public record RuleType(String name) { + + static Pattern alphaNumeric = Pattern.compile("^[a-zA-Z0-9]+$"); + + + public static RuleType fromString(String value) { + + if (!alphaNumeric.matcher(value).matches()) { + throw new IllegalArgumentException(String.format("The rule type %s must only contain numbers and letters!", value)); + } + return new RuleType(value); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleUnit.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleUnit.java new file mode 100644 index 00000000..a9cdb230 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleUnit.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.redaction.v1.server.model.drools; + +import java.util.List; + +public record RuleUnit(Integer unit, List rules) { + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index 0ad897ab..1e6ffdf5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -1,28 +1,34 @@ package com.iqser.red.service.redaction.v1.server.queue; +import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL; +import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE; +import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE; +import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER; +import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER; +import static java.lang.String.format; + +import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; + +import org.springframework.amqp.AmqpRejectAndDontRequeueException; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.exception.DroolsTimeoutException; -import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; +import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; +import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; + import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.AmqpRejectAndDontRequeueException; -import org.springframework.amqp.core.Message; -import org.springframework.amqp.rabbit.annotation.RabbitHandler; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; - -import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.*; -import static java.lang.String.format; @Slf4j @Service @@ -99,22 +105,27 @@ public class RedactionMessageReceiver { log.info(""); result.setMessageType(analyzeRequest.getMessageType()); fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result); - - } catch (Exception e) { - - if (e instanceof DroolsTimeoutException && !((DroolsTimeoutException) e).isReported()) { - rulesClient.setRulesTimeoutDetected(analyzeRequest.getDossierTemplateId()); + } catch (DroolsTimeoutException droolsTimeoutException) { + if (!droolsTimeoutException.isReported()) { + rulesClient.setRulesTimeoutDetected(analyzeRequest.getDossierTemplateId(), droolsTimeoutException.getRuleFileType()); } - - log.warn("Failed to process analyze request: {}", analyzeRequest, e); - var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); - fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); + sendAnalysisFailed(analyzeRequest, priority, droolsTimeoutException); + } catch (Exception e) { + sendAnalysisFailed(analyzeRequest, priority, e); } } + private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) { + + log.warn("Failed to process analyze request: {}", analyzeRequest, e); + var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp)); + } + + @RabbitHandler @RabbitListener(queues = REDACTION_DQL) public void receiveAnalyzeRequestDQL(Message in) throws IOException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/KieWrapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/KieWrapper.java deleted file mode 100644 index e9765698..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/KieWrapper.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; - -import org.kie.api.runtime.KieContainer; - -public record KieWrapper( KieContainer container,long rulesVersion) { - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java deleted file mode 100644 index fb09e99a..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ /dev/null @@ -1,326 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; - -import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE; - -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.RequestBody; - -import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues; -import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; -import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; -import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; -import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; -import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; -import com.iqser.red.service.redaction.v1.server.redaction.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; - -import io.micrometer.core.annotation.Timed; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.experimental.FieldDefaults; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -@RequiredArgsConstructor -public class AnalyzeService { - - private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise"; - - DictionaryService dictionaryService; - DroolsExecutionService droolsExecutionService; - EntityRedactionService entityRedactionService; - RedactionLogCreatorService redactionLogCreatorService; - RedactionStorageService redactionStorageService; - RedactionChangeLogService redactionChangeLogService; - LegalBasisClient legalBasisClient; - RedactionServiceSettings redactionServiceSettings; - ImportedRedactionService importedRedactionService; - SectionFinderService sectionFinderService; - ManualRedactionEntryService manualRedactionEntryService; - - FunctionTimerValues redactmanagerAnalyzePagewiseValues; - - - @Timed("redactmanager_analyze") - public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { - - long startTime = System.currentTimeMillis(); - - var wrapper = droolsExecutionService.getLatestKieContainer(analyzeRequest.getDossierTemplateId()); - log.info("Updated Rules to Version {} for file {} in dossier {}", wrapper.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); - log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - - List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document); - - entityRedactionService.addDictionaryEntities(dictionary, document); - log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, wrapper.container(), analyzeRequest, nerEntities); - log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, - analyzeRequest.getDossierTemplateId(), - notFoundManualRedactionEntries, - getComments(analyzeRequest)); - - List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), - analyzeRequest.getAnalysisNumber(), - redactionLogEntries, - toSimplifiedSectionText(legalBasis), - dictionary.getVersion().getDossierTemplateVersion(), - dictionary.getVersion().getDossierVersion(), - wrapper.rulesVersion(), - legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); - - List importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - redactionLog.getRedactionLogEntry(), - true); - redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries); - - return finalizeAnalysis(analyzeRequest, startTime, redactionLog, document.getNumberOfPages(), dictionary.getVersion(), false, addedFileAttributes); - } - - - private static Map> getComments(AnalyzeRequest analyzeRequest) { - - if (analyzeRequest.getManualRedactions() == null) { - return Collections.emptyMap(); - } - if (analyzeRequest.getManualRedactions().getComments() == null) { - return Collections.emptyMap(); - } - return analyzeRequest.getManualRedactions().getComments(); - } - - - @Timed("redactmanager_reanalyze") - @SneakyThrows - public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { - - long startTime = System.currentTimeMillis(); - RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - // not yet ready for reanalysis - if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) { - return analyze(analyzeRequest); - } - - DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), - new DictionaryVersion(previousRedactionLog.getDictionaryVersion(), previousRedactionLog.getDossierDictionaryVersion()), - analyzeRequest.getDossierId()); - - Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement); - List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); - log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - if (sectionsToReAnalyse.isEmpty()) { - return finalizeAnalysis(analyzeRequest, - startTime, - previousRedactionLog, - document.getNumberOfPages(), - dictionaryIncrement.getDictionaryVersion(), - true, - new HashSet<>()); - } - - NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); - log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - var wrapper = droolsExecutionService.getLatestKieContainer(analyzeRequest.getDossierTemplateId()); - log.info("Updated Rules to version {} for file {} in dossier {}", wrapper.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document); - - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); - log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, - document, - sectionsToReAnalyse, - wrapper.container(), - analyzeRequest, - nerEntities); - log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, - analyzeRequest.getDossierTemplateId(), - notFoundManualRedactionEntries, - getComments(analyzeRequest)); - - var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - newRedactionLogEntries, - false); - - previousRedactionLog.getRedactionLogEntry() - .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE)); - previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries); - - return finalizeAnalysis(analyzeRequest, - startTime, - previousRedactionLog, - document.getNumberOfPages(), - dictionaryIncrement.getDictionaryVersion(), - true, - addedFileAttributes); - } - - - private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, - long startTime, - RedactionLog redactionLog, - int numberOfPages, - DictionaryVersion dictionaryVersion, - boolean isReanalysis, - Set addedFileAttributes) { - - redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); - redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); - - excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); - - RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - redactionLog, - analyzeRequest.getAnalysisNumber()); - log.info("Created Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); - log.info("Stored Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - - long duration = System.currentTimeMillis() - startTime; - - redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration); - - return AnalyzeResult.builder() - .dossierId(analyzeRequest.getDossierId()) - .fileId(analyzeRequest.getFileId()) - .duration(duration) - .numberOfPages(numberOfPages) - .hasUpdates(redactionLogChange.isHasChanges()) - .analysisVersion(redactionServiceSettings.getAnalysisVersion()) - .analysisNumber(analyzeRequest.getAnalysisNumber()) - .rulesVersion(redactionLog.getRulesVersion()) - .dictionaryVersion(redactionLog.getDictionaryVersion()) - .legalBasisVersion(redactionLog.getLegalBasisVersion()) - .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion()) - .wasReanalyzed(isReanalysis) - .manualRedactions(analyzeRequest.getManualRedactions()) - .addedFileAttributes(addedFileAttributes) - .build(); - } - - - private static List getSectionsToReAnalyse(Document document, Set sectionsToReanalyseIds) { - - return document.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTreeId().get(0))).collect(Collectors.toList()); - } - - - private Set getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, Document document, DictionaryIncrement dictionaryIncrement) { - - return analyzeRequest.getSectionsToReanalyse().isEmpty() // - ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, redactionLog, document, analyzeRequest) // - : analyzeRequest.getSectionsToReanalyse(); - } - - - private NerEntities getEntityRecognitionEntitiesFilteredBySectionIds(AnalyzeRequest analyzeRequest, Document document, Set sectionsToReanalyseIds) { - - NerEntities nerEntities; - if (redactionServiceSettings.isNerServiceEnabled()) { - NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - nerEntitiesModel = filterNerEntitiesModelBySectionIds(sectionsToReanalyseIds, nerEntitiesModel); - nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document); - } else { - nerEntities = new NerEntities(Collections.emptyList()); - } - return nerEntities; - } - - - private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) { - - return new NerEntitiesModel(nerEntitiesModel.getData().entrySet().stream() // - .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) // - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - } - - - private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, Document document) { - - NerEntities nerEntities; - if (redactionServiceSettings.isNerServiceEnabled()) { - nerEntities = NerEntitiesAdapter.toNerEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), document); - } else { - nerEntities = new NerEntities(Collections.emptyList()); - } - return nerEntities; - } - - - public List toSimplifiedSectionText(List legalBasis) { - - return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); - } - - - private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { - - if (excludedPages != null && !excludedPages.isEmpty()) { - redactionLog.getRedactionLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { - if (excludedPages.contains(pos.getPage())) { - entry.setExcluded(true); - } - })); - } - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsSyntaxValidationFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsSyntaxValidationFactory.java deleted file mode 100644 index d8a769e3..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsSyntaxValidationFactory.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; - -import java.util.List; - -import org.kie.api.builder.KieBuilder; -import org.kie.api.builder.Message; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage; -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; - -@Service -public class DroolsSyntaxValidationFactory { - - public DroolsSyntaxValidation buildDroolsSyntaxValidation(KieBuilder kieBuilder) { - - List errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR); - List droolsSyntaxErrorMessages = errorMessages.stream().map(this::buildDroolsSyntaxErrorMessage).toList(); - return DroolsSyntaxValidation.builder().compiled(droolsSyntaxErrorMessages.isEmpty()).droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build(); - } - - - public DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) { - - return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java deleted file mode 100644 index c58fe971..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; - -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.CustomEntityCreationAdapter; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; - -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class EntityRedactionService { - - DroolsExecutionService droolsExecutionService; - EntityEnrichmentService entityEnrichmentService; - CustomEntityCreationAdapter customEntityCreationAdapter; - - - public Set addRuleEntities(Dictionary dictionary, Document document, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) { - - log.debug("Starting Drools Execution"); - long droolsStart = System.currentTimeMillis(); - List allFileAttributes = droolsExecutionService.executeRules(kieContainer, - document, - dictionary, - analyzeRequest.getFileAttributes(), - analyzeRequest.getManualRedactions(), - nerEntities); - log.debug("Finished Drools Execution in {} ms", System.currentTimeMillis() - droolsStart); - return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet()); - } - - - public Set addRuleEntities(Dictionary dictionary, - Document document, - List sectionsToReanalyze, - KieContainer kieContainer, - AnalyzeRequest analyzeRequest, - NerEntities nerEntities) { - - log.debug("Starting Drools execution"); - long ruleExecutionStart = System.currentTimeMillis(); - List allFileAttributes = droolsExecutionService.executeRules(kieContainer, - document, - sectionsToReanalyze, - dictionary, - analyzeRequest.getFileAttributes(), - analyzeRequest.getManualRedactions(), - nerEntities); - log.debug("Finished Drools execution in {} ms", System.currentTimeMillis() - ruleExecutionStart); - - return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet()); - } - - - public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { - - for (var model : dictionary.getDictionaryModels()) { - bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary()); - bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); - bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); - } - } - - - public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation, - String type, - EntityType entityType, - SemanticNode node, - boolean isDossierDictionaryEntry) { - - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); - searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) - .stream() - .filter(boundary -> entityCreationService.isValidEntityBoundary(node.getTextBlock(), boundary)) - .map(bounds -> entityCreationService.forceByBoundary(bounds, type, entityType, node)) - .peek(entity -> entity.setDictionaryEntry(true)) - .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry)) - .forEach(entity -> entity.addEngine(Engine.DICTIONARY)); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RuleBuilderModelService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RuleBuilderModelService.java deleted file mode 100644 index 18570833..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RuleBuilderModelService.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; - -import java.util.Collections; - -import org.springframework.stereotype.Service; - -import com.iqser.red.service.redaction.v1.model.RuleBuilderModel; - -@Service -public class RuleBuilderModelService { - - // TODO: Evaluate if we need to implement this or remove it entirely. - public RuleBuilderModel getRuleBuilderModel() { - - RuleBuilderModel ruleBuilderModel = new RuleBuilderModel(); - - ruleBuilderModel.setWhenClauses(Collections.emptyList()); - ruleBuilderModel.setThenConditions(Collections.emptyList()); - - return ruleBuilderModel; - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java new file mode 100644 index 00000000..16846c9e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -0,0 +1,496 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.RequestBody; + +import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; +import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; +import com.iqser.red.service.redaction.v1.server.model.KieWrapper; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; +import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter; +import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService; +import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import io.micrometer.core.annotation.Timed; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +@RequiredArgsConstructor +public class AnalyzeService { + + private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise"; + + DictionaryService dictionaryService; + EntityDroolsExecutionService entityDroolsExecutionService; + ComponentDroolsExecutionService componentDroolsExecutionService; + KieContainerCreationService kieContainerCreationService; + DictionarySearchService dictionarySearchService; + RedactionLogCreatorService redactionLogCreatorService; + EntityLogCreatorService entityLogCreatorService; + ComponentLogCreatorService componentLogCreatorService; + RedactionStorageService redactionStorageService; + RedactionChangeLogService redactionChangeLogService; + EntityChangeLogService entityChangeLogService; + LegalBasisClient legalBasisClient; + RedactionServiceSettings redactionServiceSettings; + ImportedRedactionService importedRedactionService; + SectionFinderService sectionFinderService; + ManualRedactionEntryService manualRedactionEntryService; + + FunctionTimerValues redactmanagerAnalyzePagewiseValues; + + + @Timed("redactmanager_analyze") + public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { + + long startTime = System.currentTimeMillis(); + + var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); + log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT); + log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); + log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document); + + dictionarySearchService.addDictionaryEntities(dictionary, document); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), + document, + dictionary, + analyzeRequest.getFileAttributes(), + analyzeRequest.getManualRedactions(), + nerEntities); + log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); + EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); + + return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, entityLog, + redactionLog, + document.getNumberOfPages(), dictionary.getVersion(), false, new HashSet<>(allFileAttributes)); + } + + + private EntityLog createEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + Dictionary dictionary, + KieWrapper wrapper) { + + List entityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + + List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); + + EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), + analyzeRequest.getAnalysisNumber(), + entityLogEntries, + toEntityLogLegalBasis(legalBasis), + dictionary.getVersion().getDossierTemplateVersion(), + dictionary.getVersion().getDossierVersion(), + wrapper.rulesVersion(), + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + + List importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + entityLog.getEntityLogEntry(), + true); + + entityLog.setEntityLogEntry(importedRedactionFilteredEntries); + return entityLog; + } + + + private List toEntityLogLegalBasis(List legalBasis) { + + return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); + } + + + @Timed("redactmanager_reanalyze") + @SneakyThrows + public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) { + + long startTime = System.currentTimeMillis(); + RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + ComponentLog previousComponentLog = new ComponentLog(); + + Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + // not yet ready for reanalysis + if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) { + return analyze(analyzeRequest); + } + + DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), + new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()), + analyzeRequest.getDossierId()); + + Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement); + List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); + log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + if (sectionsToReAnalyse.isEmpty()) { + return finalizeAnalysis(analyzeRequest, + startTime, + kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), + previousEntityLog, + previousRedactionLog, + document.getNumberOfPages(), + dictionaryIncrement.getDictionaryVersion(), + true, + Collections.emptySet()); + } + KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); + log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); + log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document); + + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + sectionsToReAnalyse.forEach(node -> dictionarySearchService.addDictionaryEntities(dictionary, node)); + log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), + document, + sectionsToReAnalyse, + dictionary, + analyzeRequest.getFileAttributes(), + analyzeRequest.getManualRedactions(), + nerEntities); + log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); + EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds); + + return finalizeAnalysis(analyzeRequest, + startTime, + kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), + entityLog, + redactionLog, + document.getNumberOfPages(), + dictionaryIncrement.getDictionaryVersion(), + true, + new HashSet<>(allFileAttributes)); + } + + + @Deprecated(forRemoval = true) + private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + RedactionLog previousRedactionLog, + Set sectionsToReanalyseIds) { + + List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, + analyzeRequest.getDossierTemplateId(), + notFoundManualRedactionEntries, + getComments(analyzeRequest)); + + var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + newRedactionLogEntries, + false); + + previousRedactionLog.getRedactionLogEntry() + .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)); + previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries); + return previousRedactionLog; + } + + + private EntityLog updatePreviousEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + EntityLog previousEntityLog, + Set sectionsToReanalyseIds) { + + List newEntityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + + var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + newEntityLogEntries, + false); + + previousEntityLog.getEntityLogEntry() + .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)); + previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries); + return previousEntityLog; + } + + + private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLog entityLog, + RedactionLog redactionLog, + int numberOfPages, + DictionaryVersion dictionaryVersion, + boolean isReanalysis, + Set addedFileAttributes) { + + EntityLogChanges entityLogChanges = finalizeEntityLog(analyzeRequest, entityLog, redactionLog, dictionaryVersion); + + if (entityLogChanges.isHasChanges()) { + computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges); + } + + log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + long duration = System.currentTimeMillis() - startTime; + + redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration); + + return AnalyzeResult.builder() + .dossierId(analyzeRequest.getDossierId()) + .fileId(analyzeRequest.getFileId()) + .duration(duration) + .numberOfPages(numberOfPages) + .hasUpdates(entityLogChanges.isHasChanges()) + .analysisVersion(redactionServiceSettings.getAnalysisVersion()) + .analysisNumber(analyzeRequest.getAnalysisNumber()) + .rulesVersion(entityLog.getRulesVersion()).componentRulesVersion(kieWrapperComponentRules.rulesVersion()) + .dictionaryVersion(entityLog.getDictionaryVersion()) + .legalBasisVersion(entityLog.getLegalBasisVersion()) + .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion()) + .wasReanalyzed(isReanalysis) + .manualRedactions(analyzeRequest.getManualRedactions()) + .addedFileAttributes(addedFileAttributes) + .build(); + } + + + private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, + KieWrapper kieWrapperComponentRules, + Set addedFileAttributes, + EntityLogChanges entityLogChanges) { + + if (!kieWrapperComponentRules.isPresent()) { + return; + } + + List components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), + entityLogChanges.getEntityLog(), + addedFileAttributes.stream().toList()); + log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components); + + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog); + + } + + + private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, EntityLog entityLog, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) { + + // TODO: remove redactionLog related stuff + redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); + redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); + + excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); + RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + redactionLog, + analyzeRequest.getAnalysisNumber()); + + entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); + entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); + + excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages()); + + EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + entityLog, + analyzeRequest.getAnalysisNumber()); + log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog()); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog()); + return entityLogChanges; + } + + + private static List getSectionsToReAnalyse(Document document, Set sectionsToReanalyseIds) { + + return document.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTreeId().get(0))).collect(Collectors.toList()); + } + + + private Set getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) { + + return analyzeRequest.getSectionsToReanalyse().isEmpty() // + ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) // + : analyzeRequest.getSectionsToReanalyse(); + } + + + private NerEntities getEntityRecognitionEntitiesFilteredBySectionIds(AnalyzeRequest analyzeRequest, Document document, Set sectionsToReanalyseIds) { + + NerEntities nerEntities; + if (redactionServiceSettings.isNerServiceEnabled()) { + NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + nerEntitiesModel = filterNerEntitiesModelBySectionIds(sectionsToReanalyseIds, nerEntitiesModel); + nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document); + } else { + nerEntities = new NerEntities(Collections.emptyList()); + } + return nerEntities; + } + + + private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) { + + return new NerEntitiesModel(nerEntitiesModel.getData().entrySet().stream() // + .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) // + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + + private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, Document document) { + + NerEntities nerEntities; + if (redactionServiceSettings.isNerServiceEnabled()) { + nerEntities = NerEntitiesAdapter.toNerEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), document); + } else { + nerEntities = new NerEntities(Collections.emptyList()); + } + return nerEntities; + } + + + @Deprecated(forRemoval = true) + private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + Dictionary dictionary, + KieWrapper wrapper) { + + List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, + analyzeRequest.getDossierTemplateId(), + notFoundManualRedactionEntries, + getComments(analyzeRequest)); + + List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); + RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), + analyzeRequest.getAnalysisNumber(), + redactionLogEntries, + toRedactionLogLegalBasis(legalBasis), + dictionary.getVersion().getDossierTemplateVersion(), + dictionary.getVersion().getDossierVersion(), + wrapper.rulesVersion(), + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + + List importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + redactionLog.getRedactionLogEntry(), + true); + redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries); + return redactionLog; + } + + + private static Map> getComments(AnalyzeRequest analyzeRequest) { + + if (analyzeRequest.getManualRedactions() == null) { + return Collections.emptyMap(); + } + if (analyzeRequest.getManualRedactions().getComments() == null) { + return Collections.emptyMap(); + } + return analyzeRequest.getManualRedactions().getComments(); + } + + + public List toRedactionLogLegalBasis(List legalBasis) { + + return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); + } + + + private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { + + if (excludedPages != null && !excludedPages.isEmpty()) { + redactionLog.getRedactionLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { + if (excludedPages.contains(pos.getPage())) { + entry.setExcluded(true); + } + })); + } + } + + + private void excludeExcludedPages(EntityLog entityLog, Set excludedPages) { + + if (excludedPages != null && !excludedPages.isEmpty()) { + entityLog.getEntityLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { + if (excludedPages.contains(pos.getPageNumber())) { + entry.setExcluded(true); + } + })); + } + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java new file mode 100644 index 00000000..83a500fa --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java @@ -0,0 +1,59 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.List; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators; + +@Service +public class ComponentLogCreatorService { + + public ComponentLog buildComponentLog(int analysisNumber, List components) { + + List componentLogCategories = components.stream() + .collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList()))) + .entrySet() + .stream() + .map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue())) + .toList(); + return new ComponentLog(analysisNumber, componentLogCategories, -1, -1, -1); + } + + + private ComponentLogEntry buildComponentLogEntry(Component component) { + + return ComponentLogEntry.builder() + .value(component.getValue()) + .transformation(component.getTransformation()).originalValue(component.getReferences().stream().sorted(EntityComparators.start()).map(Entity::getValue).toList()) + .componentEntityReferences(toComponentEntityReferences(component.getReferences())) + .build(); + } + + + private List toComponentEntityReferences(List references) { + + return references.stream().map(this::toComponentEntityReference).toList(); + } + + + private ComponentEntityReference toComponentEntityReference(Entity entity) { + + return ComponentEntityReference.builder() + .id(entity.getId()) + .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)) + .reason(entity.getReason()) + .ruleIdentifier(entity.getMatchedRule()) + .type(entity.getType()) + .build(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java new file mode 100644 index 00000000..2e0cbb10 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -0,0 +1,52 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DictionarySearchService { + + EntityEnrichmentService entityEnrichmentService; + + + public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { + + for (var model : dictionary.getDictionaryModels()) { + bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); + } + } + + + public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation, + String type, + EntityType entityType, + SemanticNode node, + boolean isDossierDictionaryEntry) { + + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) + .stream().filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) + .map(bounds -> entityCreationService.forceByBoundary(bounds, type, entityType, node)) + .peek(entity -> entity.setDictionaryEntry(true)) + .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry)) + .forEach(entity -> entity.addEngine(Engine.DICTIONARY)); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java similarity index 95% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index 54974fcd..9d132b63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; +package com.iqser.red.service.redaction.v1.server.service; import java.awt.Color; import java.util.ArrayList; @@ -24,15 +24,15 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntries; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryRepresentation; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.TenantDictionary; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.model.dictionary.TenantDictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryEntries; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryRepresentation; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.knecon.fforesight.tenantcommons.TenantContext; import feign.FeignException; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java new file mode 100644 index 00000000..c4d74553 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -0,0 +1,119 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class EntityChangeLogService { + + private final RedactionStorageService redactionStorageService; + + + @Timed("redactmanager_computeChanges") + public EntityLogChanges computeChanges(String dossierId, String fileId, EntityLog currentEntityLog, int analysisNumber) { + + long start = System.currentTimeMillis(); + + EntityLog previousEntityLog = redactionStorageService.getEntityLog(dossierId, fileId); + + if (previousEntityLog == null) { + currentEntityLog.getEntityLogEntry().forEach(entry -> { + entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); + }); + return new EntityLogChanges(currentEntityLog, false); + } + + List previouslyExistingEntries = previousEntityLog.getEntityLogEntry().stream().filter(entry -> !lastChangeIsRemoved(entry)).toList(); + + Map addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousEntityLog(currentEntityLog, previouslyExistingEntries); + Set removedIds = getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(currentEntityLog, previouslyExistingEntries); + + List newEntityLogEntries = previousEntityLog.getEntityLogEntry(); + + List toRemove = new ArrayList<>(); + newEntityLogEntries.forEach(entry -> { + if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now())); + var newEntry = addedEntryIds.get(entry.getId()); + newEntry.setChanges(changes); + addedEntryIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } else if (removedIds.contains(entry.getId())) { + entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now())); + } else if (addedEntryIds.containsKey(entry.getId())) { + List changes = entry.getChanges(); + changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); + var newEntry = addedEntryIds.get(entry.getId()); + newEntry.setChanges(changes); + addedEntryIds.put(entry.getId(), newEntry); + toRemove.add(entry); + } + }); + + newEntityLogEntries.removeAll(toRemove); + + addedEntryIds.forEach((k, v) -> { + if (v.getChanges().isEmpty()) { + v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); + } + newEntityLogEntries.add(v); + }); + + currentEntityLog.setEntityLogEntry(newEntityLogEntries); + + log.debug("Change computation took: {}", System.currentTimeMillis() - start); + return new EntityLogChanges(currentEntityLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty()); + } + + + private static Set getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(EntityLog currentEntityLog, List previouslyExistingEntries) { + + Set removed = new HashSet<>(previouslyExistingEntries); + currentEntityLog.getEntityLogEntry().forEach(removed::remove); + + Set removedIds = removed.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); + return removedIds; + } + + + private static Map getEntriesThatExistInCurrentButNotInPreviousEntityLog(EntityLog currentEntityLog, List previouslyExistingEntries) { + + Set currentExistingEntries = currentEntityLog.getEntityLogEntry().stream().filter(entry -> !lastChangeIsRemoved(entry)).collect(Collectors.toSet()); + + previouslyExistingEntries.forEach(currentExistingEntries::remove); + Map addedIds = new HashMap<>(); + currentExistingEntries.forEach(entry -> { + addedIds.put(entry.getId(), entry); + }); + return addedIds; + } + + + private static boolean lastChangeIsRemoved(EntityLogEntry entry) { + + return entry.getChanges().stream().reduce((a, b) -> b).map(change -> change.getType().equals(ChangeType.REMOVED)).orElse(false); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java new file mode 100644 index 00000000..e9cf4e53 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -0,0 +1,221 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +@RequiredArgsConstructor +public class EntityLogCreatorService { + + private final DictionaryService dictionaryService; + private final ManualChangeFactory manualChangeFactory; + + + public List createEntityLogEntries(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { + + List entries = new ArrayList<>(); + document.getEntities() + .stream() + .filter(EntityLogCreatorService::isEntityOrRecommendationType) + .filter(entity -> !entity.removed()) + .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); + document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); + notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); + return entries; + } + + + private static boolean isEntityOrRecommendationType(TextEntity textEntity) { + + return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION; + } + + + private List toEntityLogEntries(TextEntity textEntity, String dossierTemplateId) { + + Set processedIds = new HashSet<>(); + List redactionLogEntities = new ArrayList<>(); + + for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { + + // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows. + if (processedIds.contains(positionOnPage.getId())) { + continue; + } + processedIds.add(positionOnPage.getId()); + + EntityLogEntry redactionLogEntry = createEntityLogEntry(textEntity, dossierTemplateId); + redactionLogEntry.setId(positionOnPage.getId()); + + List rectanglesPerLine = positionOnPage.getRectanglePerLine() + .stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) + .toList(); + + redactionLogEntry.setPositions(rectanglesPerLine); + redactionLogEntities.add(redactionLogEntry); + } + + return redactionLogEntities; + } + + + private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) { + + Set referenceIds = new HashSet<>(); + entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); + int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); + boolean isHint = isHint(entity.getType(), dossierTemplateId); + return EntityLogEntry.builder() + .color(getColor(entity.getType(), dossierTemplateId, entity.applied())) + .reason(entity.buildReasonWithManualChangeDescriptions()) + .legalBasis(entity.legalBasis()) + .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) + .type(entity.getType()) + .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) + .sectionNumber(sectionNumber) + .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()).dictionaryEntry(entity.isDictionaryEntry()) + .textAfter(entity.getTextAfter()) + .textBefore(entity.getTextBefore()) + .startOffset(entity.getTextRange().start()) + .endOffset(entity.getTextRange().end()).dossierDictionaryEntry(entity.isDossierDictionaryEntry()) + .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) + .reference(referenceIds) + .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) + .state(buildEntryState(entity)) + .entryType(buildEntryType(entity, isHint)) + .build(); + } + + + public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) { + + String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); + boolean isHint = isHint(type, dossierTemplateId); + return EntityLogEntry.builder() + .id(manualEntity.getId()) + .color(getColor(type, dossierTemplateId, manualEntity.applied())) + .reason(manualEntity.buildReasonWithManualChangeDescriptions()) + .legalBasis(manualEntity.legalBasis()) + .value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue())) + .type(type) + .state(buildEntryState(manualEntity)) + .entryType(buildEntryType(manualEntity, isHint)) + .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) + .sectionNumber(0) + .matchedRule("ManualRedaction") + .dictionaryEntry(manualEntity.isDictionaryEntry()) + .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) + .textAfter("") + .textBefore("") + .startOffset(-1) + .endOffset(-1) + .positions(manualEntity.getEntityPosition().stream().map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())).toList()) + .engines(Collections.emptySet()) + .reference(Collections.emptySet()) + .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint)) + .build(); + } + + + public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) { + + String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); + boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); + return EntityLogEntry.builder() + .id(image.getId()) + .color(getColor(imageType, dossierTemplateId, image.applied())) + .type(imageType) + .reason(image.buildReasonWithManualChangeDescriptions()) + .legalBasis(image.legalBasis()) + .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(false).positions(List.of(new Position(image.getPosition(), image.getPage().getNumber()))) + .sectionNumber(image.getTreeId().get(0)) + .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) + .imageHasTransparency(image.isTransparent()) + .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) + .state(buildEntryState(image)) + .entryType(buildEntryType(image, isHint)) + .build(); + + } + + + private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { + + if (!isRedaction && !isHint(type, dossierTemplateId)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); + } + + + private boolean isHint(String type, String dossierTemplateId) { + + return dictionaryService.isHint(type, dossierTemplateId); + } + + + private EntryState buildEntryState(IEntity entity) { + + if (entity.applied()) { + return EntryState.APPLIED; + } else if (entity.skipped()) { + return EntryState.SKIPPED; + } else if (entity.ignored()) { + return EntryState.IGNORED; + } else { + return EntryState.REMOVED; + } + } + + + private EntryType buildEntryType(IEntity entity, boolean isHint) { + + if (entity instanceof TextEntity textEntity) { + return getEntryType(isHint, textEntity.getEntityType()); + } else if (entity instanceof ManualEntity manualEntity) { + if (((ManualEntity) entity).isRectangle()) { + return EntryType.AREA; + } + return getEntryType(isHint, manualEntity.getEntityType()); + } else if (entity instanceof Image) { + return EntryType.IMAGE; + } + throw new UnsupportedOperationException("Entity subclass %s not implemented!"); + } + + + private static EntryType getEntryType(boolean isHint, EntityType entityType) { + + return switch (entityType) { + case ENTITY -> isHint ? EntryType.HINT : EntryType.ENTITY; + case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; + case RECOMMENDATION -> EntryType.RECOMMENDATION; + case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; + }; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java similarity index 56% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java index ce56d4ce..19574ce6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java @@ -1,10 +1,13 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; +package com.iqser.red.service.redaction.v1.server.service; import java.util.HashSet; import java.util.List; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; @@ -25,6 +28,7 @@ public class ImportedRedactionService { private final RedactionStorageService redactionStorageService; + @Deprecated(forRemoval = true) @Timed("redactmanager_processImportedRedactions") public List processImportedRedactions(String dossierTemplateId, String dossierId, @@ -47,6 +51,29 @@ public class ImportedRedactionService { } + @Timed("redactmanager_processImportedRedactions") + public List processImportedEntities(String dossierTemplateId, + String dossierId, + String fileId, + List redactionLogEntries, + boolean addImportedRedactions) { + + var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId); + if (importedRedactions == null) { + return redactionLogEntries; + } + + redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions)); + + if (addImportedRedactions) { + return addImportedRedactionsEntityLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions); + } + + return redactionLogEntries; + } + + + @Deprecated(forRemoval = true) private List addImportedRedactionsRedactionLogEntries(String dossierTemplateId, List redactionLogEntries, ImportedRedactions importedRedactions) { @@ -70,6 +97,40 @@ public class ImportedRedactionService { } + private List addImportedRedactionsEntityLogEntries(String dossierTemplateId, List entityLogEntries, ImportedRedactions importedRedactions) { + + for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) { + for (var importedRedaction : importedRedactionsValues) { + EntityLogEntry redactionLogEntry = EntityLogEntry.builder() + .id(importedRedaction.getId()) + .type(IMPORTED_REDACTION_TYPE) + .imported(true) + .state(EntryState.APPLIED) + .positions(toPositions(importedRedaction.getPositions())) + .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId)) + .build(); + + entityLogEntries.add(redactionLogEntry); + } + } + + return entityLogEntries; + } + + + private List toPositions(List positions) { + + return positions.stream().map(this::toPosition).toList(); + } + + + private Position toPosition(Rectangle rectangle) { + + return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage()); + } + + + @Deprecated(forRemoval = true) private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) { for (Rectangle rectangle : redactionLogEntry.getPositions()) { @@ -91,6 +152,27 @@ public class ImportedRedactionService { } + private void addIntersections(EntityLogEntry redactionLogEntry, ImportedRedactions importedRedactions) { + + for (Position rectangle : redactionLogEntry.getPositions()) { + var normalizedRectangle = normalize(rectangle); + if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) { + var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber()); + for (ImportedRedaction importedRedaction : importedRedactionsOnPage) { + for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) { + if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) { + if (redactionLogEntry.getImportedRedactionIntersections() == null) { + redactionLogEntry.setImportedRedactionIntersections(new HashSet<>()); + } + redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId()); + } + } + } + } + } + } + + boolean valueInRange(float value, float min, float max) { return round(value) >= round(min) && round(value) <= round(max); @@ -111,6 +193,7 @@ public class ImportedRedactionService { } + @Deprecated(forRemoval = true) private Rectangle normalize(Rectangle rectangle) { Rectangle r = new Rectangle(); @@ -138,6 +221,33 @@ public class ImportedRedactionService { } + private Rectangle normalize(Position position) { + + Rectangle r = new Rectangle(); + Point p = new Point(); + + if (position.getRectangle()[2] < 0) { + p.setX(position.getRectangle()[0] + position.getRectangle()[2]); + r.setWidth(Math.abs(position.getRectangle()[2])); + } else { + p.setX(position.getRectangle()[0]); + r.setWidth(position.getRectangle()[2]); + } + + if (position.getRectangle()[3] < 0) { + p.setY(position.getRectangle()[1] + position.getRectangle()[3]); + r.setHeight(Math.abs(position.getRectangle()[3])); + } else { + p.setY(position.getRectangle()[1]); + r.setHeight(position.getRectangle()[3]); + } + + r.setTopLeft(p); + r.setPage(position.getPageNumber()); + return r; + } + + private float round(float value) { double d = Math.pow(10, 0); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java similarity index 95% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java index 5a244e9a..0f03176a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualChangeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; +package com.iqser.red.service.redaction.v1.server.service; import java.time.OffsetDateTime; import java.util.List; @@ -12,8 +12,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; @Service public class ManualChangeFactory { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java similarity index 83% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualChangesApplicationService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index 74836290..e89b7489 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.services; +package com.iqser.red.service.redaction.v1.server.service; import java.awt.geom.Rectangle2D; import java.util.HashSet; @@ -7,13 +7,14 @@ import java.util.NoSuchElementException; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.Entity; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.PositionOnPage; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import lombok.RequiredArgsConstructor; @@ -23,14 +24,14 @@ public class ManualChangesApplicationService { private final EntityCreationService entityCreationService; - public void recategorize(Entity entityToBeReCategorized, ManualRecategorization manualRecategorization) { + public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) { - if (entityToBeReCategorized instanceof Image image) { + if (IEntityToBeReCategorized instanceof Image image) { image.setImageType(ImageType.fromString(manualRecategorization.getType())); return; } // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed! - if (entityToBeReCategorized instanceof TextEntity textEntity) { + if (IEntityToBeReCategorized instanceof TextEntity textEntity) { TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode()); recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage()); recategorizedEntity.getManualOverwrite().addChange(manualRecategorization); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java similarity index 98% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java index 1712f5ec..74c4dfc4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; +package com.iqser.red.service.redaction.v1.server.service; import java.time.OffsetDateTime; import java.util.ArrayList; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java similarity index 82% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index cb39094b..ddcc9dcd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; +package com.iqser.red.service.redaction.v1.server.service; import java.awt.geom.Rectangle2D; import java.util.ArrayList; @@ -7,23 +7,28 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.PositionOnPage; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; -import com.iqser.red.service.redaction.v1.server.redaction.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -146,13 +151,30 @@ public class RedactionLogCreatorService { .startOffset(entity.getTextRange().start()) .endOffset(entity.getTextRange().end()) .isDossierDictionaryEntry(entity.isDossierDictionaryEntry()) - .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) + .engines(getEngines(entity)) .reference(referenceIds) - .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) + .manualChanges(mapManualChanges(entity.getManualOverwrite(), isHint)) .build(); } + private Set getEngines(TextEntity entity) { + + return entity.getEngines() != null ? mapToEngines(entity.getEngines()) : Collections.emptySet(); + } + + + private Set mapToEngines(Set engines) { + + return engines.stream().map(engine -> switch (engine) { + case NER -> Engine.NER; + case RULE -> Engine.RULE; + case DICTIONARY -> Engine.DICTIONARY; + default -> null; + }).filter(Objects::nonNull).collect(Collectors.toSet()); + } + + public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map> comments) { String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); @@ -184,12 +206,26 @@ public class RedactionLogCreatorService { .collect(Collectors.toList())) .engines(Collections.emptySet()) .reference(Collections.emptySet()) - .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint)) + .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint)) .comments(buildRedactionLogComments(comments, manualEntity.getId())) .build(); } + private List mapManualChanges(ManualChangeOverwrite manualEntity, boolean isHint) { + + return manualChangeFactory.toManualChangeList(manualEntity.getManualChangeLog(), isHint).stream().map(this::mapManualChange).toList(); + } + + + private ManualChange mapManualChange(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange manualChange) { + + ManualChange manualChange1 = new ManualChange(); + BeanUtils.copyProperties(manualChange, manualChange1); + return manualChange1; + } + + public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map> comments) { String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); @@ -210,7 +246,7 @@ public class RedactionLogCreatorService { .sectionNumber(image.getTreeId().get(0)) .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) .imageHasTransparency(image.isTransparent()) - .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) + .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint)) .comments(buildRedactionLogComments(comments, image.getId())) .build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java new file mode 100644 index 00000000..dd948fc7 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java @@ -0,0 +1,131 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.kie.api.runtime.KieSession; + +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.utils.DateConverter; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class ComponentCreationService { + + KieSession kieSession; + Set referencedEntities = new HashSet<>(); + + + public void firstOrElse(String ruleIdentifier, String category, Collection entities, String fallback) { + + String transformation = String.format("First found value or else '%s'", fallback); + String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback); + create(ruleIdentifier, category, value, transformation, entities); + } + + + public void joining(String ruleIdentifier, String category, Collection entities) { + + joining(ruleIdentifier, category, entities, ", "); + } + + + public void joining(String ruleIdentifier, String category, Collection entities, String delimiter) { + + String transformation = String.format("Joining all values with '%s'", delimiter); + String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter)); + create(ruleIdentifier, category, value, transformation, entities); + } + + + public void joiningUnique(String ruleIdentifier, String category, Collection entities) { + + joining(ruleIdentifier, category, entities, ", "); + } + + + public void joiningUnique(String ruleIdentifier, String category, Collection entities, String delimiter) { + + String transformation = String.format("Joining all values with '%s'", delimiter); + String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter)); + create(ruleIdentifier, category, value, transformation, entities); + } + + + public void convertDates(String ruleIdentifier, String category, Collection entities) { + + convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy"); + } + + + public void convertDates(String ruleIdentifier, String category, Collection entities, String resultFormat) { + + String transformation = "Convert values of type to dd/MM/yyyy joined with ', '"; + String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", ")); + create(ruleIdentifier, category, date, transformation, entities); + } + + + public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection entities) { + + entities.forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity))); + } + + + public void create(String ruleIdentifier, String category, String value, String transformation, Collection references) { + + referencedEntities.addAll(references); + + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .category(category) + .value(value) + .transformation(transformation) + .references(new LinkedList<>(references))); + } + + + public void create(String ruleIdentifier, String category, String value, String transformation, Entity reference) { + + referencedEntities.add(reference); + List referenceList = new LinkedList<>(); + referenceList.add(reference); + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .category(category) + .value(value) + .transformation(transformation) + .references(referenceList) + .build()); + } + + + public void create(String ruleIdentifier, String category, String value, String transformation) { + + create(ruleIdentifier, category, value, transformation, Collections.emptyList()); + } + + + public void create(String ruleIdentifier, String category, String value) { + + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .category(category) + .value(value) + .transformation("") + .references(Collections.emptyList()) + .build()); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java similarity index 86% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java index 5f3444d4..3578128b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.data.mapper; +package com.iqser.red.service.redaction.v1.server.service.document; import java.util.Arrays; import java.util.HashSet; @@ -7,22 +7,22 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Footer; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Header; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; -import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Headline; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentData; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java new file mode 100644 index 00000000..7364bcdd --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import java.util.Comparator; + +import com.iqser.red.service.redaction.v1.server.model.component.Entity; + +public abstract class EntityComparators implements Comparator { + + private static class LongestEntity implements Comparator { + + @Override + public int compare(Entity Entity, Entity otherEntity) { + + return Integer.compare(Entity.getLength(), otherEntity.getLength()); + } + + } + + private static class FirstEntity implements Comparator { + + @Override + public int compare(Entity Entity, Entity otherEntity) { + + return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset()); + } + + } + + + public static Comparator length() { + + return new LongestEntity(); + } + + + public static Comparator start() { + + return new FirstEntity(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java similarity index 76% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index a39ae9b8..d3804b6e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -1,6 +1,6 @@ -package com.iqser.red.service.redaction.v1.server.document.services; +package com.iqser.red.service.redaction.v1.server.service.document; -import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; +import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import java.util.Collection; import java.util.Collections; @@ -17,26 +17,25 @@ import org.apache.commons.lang3.tuple.Pair; import org.kie.api.runtime.KieSession; import com.google.common.base.Functions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.ConsecutiveBoundaryCollector; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ManualChangeOverwrite; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.PositionOnPage; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.utils.IdBuilder; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -58,144 +57,144 @@ public class EntityCreationService { public Stream betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { - List startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock()); - List stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock()); + List startTextRanges = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock()); + List stopTextRanges = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock()); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node); } public Stream betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) { - List startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock()); - List stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock()); + List startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock()); + List stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock()); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } public Stream betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node) { - List startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock()); - List stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock()); + List startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock()); + List stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock()); - startBoundaries.forEach(boundary -> { - boundary.setStart(boundary.start() - start.length()); - boundary.setEnd(boundary.end() - start.length()); + startBoundaries.forEach(textRange -> { + textRange.setStart(textRange.start() - start.length()); + textRange.setEnd(textRange.end() - start.length()); }); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } public Stream betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) { - List startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock()); - List stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock()); + List startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock()); + List stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock()); - startBoundaries.forEach(boundary -> { - boundary.setStart(boundary.start() - start.length()); - boundary.setEnd(boundary.end() - start.length()); + startBoundaries.forEach(textRange -> { + textRange.setStart(textRange.start() - start.length()); + textRange.setEnd(textRange.end() - start.length()); }); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } public Stream betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) { - List startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock()); - List stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock()); + List startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock()); + List stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock()); - stopBoundaries.forEach(boundary -> { - boundary.setStart(boundary.start() + stop.length()); - boundary.setEnd(boundary.end() + stop.length()); + stopBoundaries.forEach(textRange -> { + textRange.setStart(textRange.start() + stop.length()); + textRange.setEnd(textRange.end() + stop.length()); }); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } public Stream betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) { - List startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock()); - List stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock()); + List startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock()); + List stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock()); - stopBoundaries.forEach(boundary -> { - boundary.setStart(boundary.start() + stop.length()); - boundary.setEnd(boundary.end() + stop.length()); + stopBoundaries.forEach(textRange -> { + textRange.setStart(textRange.start() + stop.length()); + textRange.setEnd(textRange.end() + stop.length()); }); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } public Stream betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) { - List startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock()); - List stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock()); + List startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock()); + List stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock()); - startBoundaries.forEach(boundary -> { - boundary.setStart(boundary.start() - start.length()); - boundary.setEnd(boundary.end() - start.length()); + startBoundaries.forEach(textRange -> { + textRange.setStart(textRange.start() - start.length()); + textRange.setEnd(textRange.end() - start.length()); }); - stopBoundaries.forEach(boundary -> { - boundary.setStart(boundary.start() + stop.length()); - boundary.setEnd(boundary.end() + stop.length()); + stopBoundaries.forEach(textRange -> { + textRange.setStart(textRange.start() + stop.length()); + textRange.setEnd(textRange.end() + stop.length()); }); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } public Stream betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) { - List startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock()); - List stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock()); + List startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock()); + List stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock()); - startBoundaries.forEach(boundary -> { - boundary.setStart(boundary.start() - start.length()); - boundary.setEnd(boundary.end() - start.length()); + startBoundaries.forEach(textRange -> { + textRange.setStart(textRange.start() - start.length()); + textRange.setEnd(textRange.end() - start.length()); }); - stopBoundaries.forEach(boundary -> { - boundary.setStart(boundary.start() + stop.length()); - boundary.setEnd(boundary.end() + stop.length()); + stopBoundaries.forEach(textRange -> { + textRange.setStart(textRange.start() + stop.length()); + textRange.setEnd(textRange.end() + stop.length()); }); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } public Stream betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.getTextBlock(); - List startBoundaries = RedactionSearchUtility.findBoundariesByRegex(regexStart, textBlock); - List stopBoundaries = RedactionSearchUtility.findBoundariesByRegex(regexStop, textBlock); + List startBoundaries = RedactionSearchUtility.findTextRangesByRegex(regexStart, textBlock); + List stopBoundaries = RedactionSearchUtility.findTextRangesByRegex(regexStop, textBlock); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } public Stream betweenRegexesIgnoreCase(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.getTextBlock(); - List startBoundaries = RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexStart, 0, textBlock); - List stopBoundaries = RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexStop, 0, textBlock); + List startBoundaries = RedactionSearchUtility.findTextRangesByRegexIgnoreCase(regexStart, 0, textBlock); + List stopBoundaries = RedactionSearchUtility.findTextRangesByRegexIgnoreCase(regexStop, 0, textBlock); - return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } - public Stream betweenBoundaries(List startBoundaries, List stopBoundaries, String type, EntityType entityType, SemanticNode node) { + public Stream betweenTextRanges(List startBoundaries, List stopBoundaries, String type, EntityType entityType, SemanticNode node) { if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) { return Stream.empty(); } List entityBoundaries = findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(startBoundaries, stopBoundaries); return entityBoundaries.stream() - .map(boundary -> boundary.trim(node.getTextBlock())) - .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) - .map(boundary -> byTextRange(boundary, type, entityType, node)) + .map(textRange -> textRange.trim(node.getTextBlock())) + .filter(textRange -> isValidEntityTextRange(node.getTextBlock(), textRange)) + .map(textRange -> byTextRange(textRange, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); } @@ -205,27 +204,26 @@ public class EntityCreationService { List entityBoundaries = new LinkedList<>(); for (TextRange startTextRange : startBoundaries) { - Optional optionalStopBoundaryWithMinimalDistance = stopBoundaries.stream() + Optional optionalStopTextRangeWithMinimalDistance = stopBoundaries.stream() .filter(stopBoundary -> stopBoundary.start() > startTextRange.end()) .min(Comparator.comparingInt(TextRange::start)); - if (optionalStopBoundaryWithMinimalDistance.isEmpty()) { + if (optionalStopTextRangeWithMinimalDistance.isEmpty()) { break; } - entityBoundaries.add(new TextRange(startTextRange.end(), optionalStopBoundaryWithMinimalDistance.get().start())); + entityBoundaries.add(new TextRange(startTextRange.end(), optionalStopTextRangeWithMinimalDistance.get().start())); } return removeOuterOverlappingBoundaries(entityBoundaries); } - private static List removeOuterOverlappingBoundaries(List entityBoundaries) { + private static List removeOuterOverlappingBoundaries(List entityTextRanges) { /* In some cases we get boundaries, where one contains the other. This happens for Example when we have two start boundaries and one stop boundary after the two start boundaries. Then we get two boundaries where one is entirely contained in the other. So we want to remove the outer boundary. For Example consider the text: "a this is some text. a here is more text b". If "a" is the start string and "b" is the stop string, there are two possibilities. "this is some text. a here is more text" and "here is more text". We only want to keep the latter. */ - return entityBoundaries.stream() - .filter(boundary -> entityBoundaries.stream().noneMatch(innerBoundary -> innerBoundary != boundary && innerBoundary.containedBy(boundary))) + return entityTextRanges.stream().filter(boundary -> entityTextRanges.stream().noneMatch(innerBoundary -> innerBoundary != boundary && innerBoundary.containedBy(boundary))) .toList(); } @@ -234,8 +232,7 @@ public class EntityCreationService { public Stream bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) { return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) - .stream() - .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) + .stream().filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary)) .map(bounds -> byTextRange(bounds, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -247,9 +244,7 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); SearchImplementation searchImplementation = new SearchImplementation(strings, false); return searchImplementation.getBoundaries(textBlock, node.getTextRange()) - .stream() - .map(boundary -> toLineAfterBoundary(textBlock, boundary)) - .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) + .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -261,9 +256,7 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); SearchImplementation searchImplementation = new SearchImplementation(strings, true); return searchImplementation.getBoundaries(textBlock, node.getTextRange()) - .stream() - .map(boundary -> toLineAfterBoundary(textBlock, boundary)) - .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) + .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -273,10 +266,8 @@ public class EntityCreationService { public Stream lineAfterString(String string, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.getTextBlock(); - return RedactionSearchUtility.findBoundariesByString(string, textBlock) - .stream() - .map(boundary -> toLineAfterBoundary(textBlock, boundary)) - .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) + return RedactionSearchUtility.findTextRangesByString(string, textBlock) + .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -286,10 +277,8 @@ public class EntityCreationService { public Stream lineAfterStringIgnoreCase(String string, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.getTextBlock(); - return RedactionSearchUtility.findBoundariesByStringIgnoreCase(string, textBlock) - .stream() - .map(boundary -> toLineAfterBoundary(textBlock, boundary)) - .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) + return RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, textBlock) + .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -298,8 +287,7 @@ public class EntityCreationService { public Stream lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) { - return tableNode.streamTableCells() - .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findBoundariesByString(string, tableCell.getTextBlock()), + return tableNode.streamTableCells().flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()), tableCell, type, entityType, @@ -310,7 +298,7 @@ public class EntityCreationService { public Stream lineAfterStringAcrossColumnsIgnoreCase(String string, String type, EntityType entityType, Table tableNode) { return tableNode.streamTableCells() - .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findBoundariesByStringIgnoreCase(string, tableCell.getTextBlock()), + .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, tableCell.getTextBlock()), tableCell, type, entityType, @@ -321,23 +309,23 @@ public class EntityCreationService { /** * Looks across the remaining table row to the right of the provided TableCell if any line intersects the y coordinates of the found text. * - * @param boundaries a list of boundaries + * @param TextRanges a list of textRanges * @param tableCell the table cell * @param type the type * @param entityType the entity type * @param tableNode the table node * @return a stream of RedactionEntities */ - private Stream lineAfterBoundariesAcrossColumns(List boundaries, TableCell tableCell, String type, EntityType entityType, Table tableNode) { + private Stream lineAfterBoundariesAcrossColumns(List TextRanges, TableCell tableCell, String type, EntityType entityType, Table tableNode) { - return boundaries.stream() + return TextRanges.stream() .map(boundary -> RectangleTransformations.rectangle2DBBox(tableCell.getTextBlock().getPositions(boundary))) .map(bBox -> Pair.of(bBox.getMaxY(), bBox.getMinY())) .map(maxMinPair -> tableNode.streamRow(tableCell.getRow()) .filter(nextTableCell -> nextTableCell.getCol() > tableCell.getCol()) - .map(nextTableCell -> RedactionSearchUtility.findBoundaryOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock())) + .map(nextTableCell -> RedactionSearchUtility.findTextRangesOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock())) .map(b -> b.trim(tableNode.getTextBlock())) - .filter(boundary -> isValidEntityBoundary(tableNode.getTextBlock(), boundary)) + .filter(boundary -> isValidEntityTextRange(tableNode.getTextBlock(), boundary)) .map(boundary -> byTextRange(boundary, type, entityType, tableNode)) .filter(Optional::isPresent) .map(Optional::get)) @@ -348,12 +336,15 @@ public class EntityCreationService { public Optional semanticNodeAfterString(SemanticNode semanticNode, String string, String type, EntityType entityType) { var textBlock = semanticNode.getTextBlock(); - int startIndex = Math.min(textBlock.indexOf(string), 0); + int startIndex = textBlock.indexOf(string); + if (startIndex == -1) { + return Optional.empty(); + } var boundary = new TextRange(startIndex, semanticNode.getTextRange().end()); if (boundary.length() > 0) { boundary = new TextRange(boundary.start(), boundary.end() - 1); } - if (!isValidEntityBoundary(textBlock, boundary)) { + if (!isValidEntityTextRange(textBlock, boundary)) { return Optional.empty(); } return byTextRange(boundary, type, entityType, semanticNode); @@ -386,7 +377,7 @@ public class EntityCreationService { public Stream byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock()) + return RedactionSearchUtility.findTextRangesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock()) .stream() .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -396,7 +387,7 @@ public class EntityCreationService { public Stream byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock()) + return RedactionSearchUtility.findTextRangesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock()) .stream() .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -406,7 +397,7 @@ public class EntityCreationService { public Stream byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock()) + return RedactionSearchUtility.findTextRangesByRegex(regexPattern, group, node.getTextBlock()) .stream() .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -416,7 +407,7 @@ public class EntityCreationService { public Stream byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexPattern, group, node.getTextBlock()) + return RedactionSearchUtility.findTextRangesByRegexIgnoreCase(regexPattern, group, node.getTextBlock()) .stream() .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -426,7 +417,7 @@ public class EntityCreationService { public Stream byString(String keyword, String type, EntityType entityType, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock()) + return RedactionSearchUtility.findTextRangesByString(keyword, node.getTextBlock()) .stream() .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -436,7 +427,7 @@ public class EntityCreationService { public Stream byStringIgnoreCase(String keyword, String type, EntityType entityType, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByStringIgnoreCase(keyword, node.getTextBlock()) + return RedactionSearchUtility.findTextRangesByStringIgnoreCase(keyword, node.getTextBlock()) .stream() .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -479,7 +470,7 @@ public class EntityCreationService { if (textRange.length() > 0) { textRange = new TextRange(textRange.start(), textRange.end() - 1); } - if (!isValidEntityBoundary(node.getTextBlock(), textRange)) { + if (!isValidEntityTextRange(node.getTextBlock(), textRange)) { return Optional.empty(); } return byTextRange(textRange, type, entityType, node); @@ -647,7 +638,7 @@ public class EntityCreationService { } - public boolean isValidEntityBoundary(TextBlock textBlock, TextRange textRange) { + public boolean isValidEntityTextRange(TextBlock textBlock, TextRange textRange) { return textRange.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, textRange); } @@ -719,7 +710,11 @@ public class EntityCreationService { } - private static TextRange toLineAfterBoundary(TextBlock textBlock, TextRange textRange) { + private static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) { + + if (textBlock.getTextRange().end() == textRange.end()) { + return new TextRange(textRange.end(), textRange.end()); + } return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java similarity index 91% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java index 926fff44..586ab810 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.services; +package com.iqser.red.service.redaction.v1.server.service.document; import java.util.Arrays; import java.util.List; @@ -6,9 +6,9 @@ import java.util.Objects; import org.springframework.stereotype.Service; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import lombok.RequiredArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/CustomEntityCreationAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java similarity index 90% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/CustomEntityCreationAdapter.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java index e4fdf7c7..28aea8a8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/CustomEntityCreationAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.adapter; +package com.iqser.red.service.redaction.v1.server.service.document; import static java.lang.String.format; import static java.util.stream.Collectors.groupingBy; @@ -20,30 +20,28 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.PositionOnPage; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.redaction.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.redaction.model.RectangleWithPage; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import lombok.extern.slf4j.Slf4j; @Slf4j @Service -public class CustomEntityCreationAdapter { +public class ManualEntityCreationService { - private static final double MATCH_THRESHOLD = 5; // sum of distances for each corner + private static final double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities private final EntityCreationService entityCreationService; @Autowired - public CustomEntityCreationAdapter(EntityEnrichmentService entityEnrichmentService) { + public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService) { entityCreationService = new EntityCreationService(entityEnrichmentService); } @@ -100,7 +98,7 @@ public class CustomEntityCreationAdapter { TextEntity correctEntity = entityCreationService.forceByBoundary(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node); if (manualEntity.isApplied()) { - correctEntity.force(manualEntity.getRuleIdentifier(), manualEntity.getReason(), manualEntity.getLegalBasis()); + correctEntity.apply(manualEntity.getRuleIdentifier(), manualEntity.getReason(), manualEntity.getLegalBasis()); } else { correctEntity.skip(manualEntity.getRuleIdentifier(), manualEntity.getReason()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionEntryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java similarity index 81% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionEntryService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java index 1dc59c7e..366acff3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionEntryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; +package com.iqser.red.service.redaction.v1.server.service.document; import java.util.Collection; import java.util.Collections; @@ -10,9 +10,8 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.CustomEntityCreationAdapter; -import com.iqser.red.service.redaction.v1.server.redaction.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -22,14 +21,14 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class ManualRedactionEntryService { - private final CustomEntityCreationAdapter customEntityCreationAdapter; + private final ManualEntityCreationService manualEntityCreationService; public List addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document) { List notFoundManualRedactionEntries = Collections.emptyList(); if (analyzeRequest.getManualRedactions() != null) { - notFoundManualRedactionEntries = customEntityCreationAdapter.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions() + notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions() .getEntriesToAdd(), document); log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java similarity index 95% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java index fd6139e6..5d4096fe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.adapter; +package com.iqser.red.service.redaction.v1.server.service.document; import java.util.Collection; import java.util.Comparator; @@ -9,10 +9,11 @@ import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.AccessLevel; import lombok.experimental.FieldDefaults; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/PropertiesMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java similarity index 83% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/PropertiesMapper.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java index d6b0fa31..220a35de 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/PropertiesMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java @@ -1,18 +1,16 @@ -package com.iqser.red.service.redaction.v1.server.document.data.mapper; +package com.iqser.red.service.redaction.v1.server.service.document; import java.awt.geom.Rectangle2D; import java.util.Arrays; import java.util.List; import java.util.Map; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; -import lombok.AccessLevel; -import lombok.experimental.FieldDefaults; import lombok.experimental.UtilityClass; @UtilityClass diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java similarity index 85% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index 067a0229..3eeab748 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; +package com.iqser.red.service.redaction.v1.server.service.document; import java.util.Collection; import java.util.Collections; @@ -8,9 +8,11 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; @@ -19,13 +21,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -34,18 +34,18 @@ import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j -@Component +@Service @RequiredArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -class SectionFinderService { +public class SectionFinderService { @Timed("redactmanager_findSectionsToReanalyse") - public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Document document, AnalyzeRequest analyzeRequest) { + public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) { long start = System.currentTimeMillis(); Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set sectionsToReanalyse = new HashSet<>(); - for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { + for (EntityLogEntry entry : entityLog.getEntityLogEntry()) { if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { sectionsToReanalyse.add(entry.getSectionNumber()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java new file mode 100644 index 00000000..e85d09a0 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java @@ -0,0 +1,93 @@ +package com.iqser.red.service.redaction.v1.server.service.drools; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.kie.api.runtime.KieContainer; +import org.kie.api.runtime.KieSession; +import org.kie.api.runtime.rule.QueryResults; +import org.kie.api.runtime.rule.QueryResultsRow; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +public class ComponentDroolsExecutionService { + + RedactionServiceSettings settings; + + + public List executeRules(KieContainer kieContainer, EntityLog entityLog, List fileAttributes) { + + KieSession kieSession = kieContainer.newKieSession(); + ComponentCreationService componentCreationService = new ComponentCreationService(kieSession); + + kieSession.setGlobal("componentCreationService", componentCreationService); + entityLog.getEntityLogEntry().stream().map(Entity::fromEntityLogEntry).forEach(kieSession::insert); + fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert); + + CompletableFuture completableFuture = CompletableFuture.supplyAsync(() -> { + kieSession.fireAllRules(); + return null; + }); + kieSession.halt(); + try { + completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get(); + } catch (ExecutionException e) { + kieSession.dispose(); + if (e.getCause() instanceof TimeoutException) { + throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT); + } + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + List resultingFileAttributes = getFileAttributes(kieSession); + List components = getComponents(kieSession); + kieSession.dispose(); + return components; + } + + + public List getFileAttributes(KieSession kieSession) { + + List fileAttributes = new LinkedList<>(); + QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes"); + for (QueryResultsRow resultsRow : entitiesResult) { + fileAttributes.add((FileAttribute) resultsRow.get("$fileAttribute")); + } + return fileAttributes; + } + + + public List getComponents(KieSession kieSession) { + + List components = new LinkedList<>(); + QueryResults entitiesResult = kieSession.getQueryResults("getComponents"); + for (QueryResultsRow resultsRow : entitiesResult) { + components.add((Component) resultsRow.get("$component")); + } + return components; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java new file mode 100644 index 00000000..2242a91b --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java @@ -0,0 +1,122 @@ +package com.iqser.red.service.redaction.v1.server.service.drools; + +import java.util.List; +import java.util.stream.Collectors; + +import org.kie.api.builder.KieBuilder; +import org.kie.api.builder.Message; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage; +import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; +import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; +import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +@Service +@RequiredArgsConstructor +public class DroolsSyntaxValidationService { + + private final KieContainerCreationService kieContainerCreationService; + + + @SneakyThrows + public DroolsSyntaxValidation testRules(RuleValidationModel rules) { + + DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType())); + DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules); + droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages()); + return droolsCompilerSyntaxValidation; + } + + + private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) { + + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); + DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation(); + + RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { + case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString()); + case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + }; + + if (!ruleFileBluePrint.getImports().equals(baseRuleFileBluePrint.getImports())) { + customSyntaxValidation.getDroolsSyntaxErrorMessages() + .add(DroolsSyntaxErrorMessage.builder() + .line(ruleFileBluePrint.getImportLine()) + .column(0) + .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports())) + .build()); + } + if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) { + customSyntaxValidation.getDroolsSyntaxErrorMessages() + .add(DroolsSyntaxErrorMessage.builder().line(ruleFileBluePrint.getGlobalsLine()) + .column(0).message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals())) + .build()); + } + baseRuleFileBluePrint.getQueries().forEach(basicQuery -> { + if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) { + customSyntaxValidation.getDroolsSyntaxErrorMessages() + .add(DroolsSyntaxErrorMessage.builder() + .line(basicQuery.getLine()) + .column(0) + .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode())) + .build()); + } + }); + baseRuleFileBluePrint.streamAllRules().forEach(basicRule -> { + if (!validateRuleIsPresent(basicRule, ruleFileBluePrint)) { + customSyntaxValidation.getDroolsSyntaxErrorMessages() + .add(DroolsSyntaxErrorMessage.builder() + .line(basicRule.getLine()) + .column(0) + .message(String.format("Changing or removing the rule %s is not allowed! Must be: %n%s", basicRule.getName(), basicRule.getCode())) + .build()); + } + }); + return customSyntaxValidation; + } + + + private static boolean validateRuleIsPresent(BasicRule basicRule, RuleFileBluePrint ruleFileBluePrint) { + + return ruleFileBluePrint.findRulesByIdentifier(basicRule.getIdentifier()).stream().anyMatch(otherRule -> otherRule.getCode().equals(basicRule.getCode())); + } + + + private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) { + + return ruleFileBluePrint.getQueries().stream().anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode())); + } + + + private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(RuleValidationModel rules) { + + var versionId = System.currentTimeMillis(); + var testRules = "test-rules"; + KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules, + versionId, rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType())); + return buildDroolsCompilerSyntaxValidation(kieBuilder); + } + + + private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) { + + List errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR); + List droolsSyntaxErrorMessages = errorMessages.stream().map(this::buildDroolsSyntaxErrorMessage).collect(Collectors.toList()); + return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build(); + } + + + private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) { + + return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java similarity index 52% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index 02f2fa84..641d5bf4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -1,41 +1,30 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; +package com.iqser.red.service.redaction.v1.server.service.drools; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; -import java.util.concurrent.*; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; -import com.google.common.util.concurrent.SimpleTimeLimiter; -import com.iqser.red.service.redaction.v1.server.exception.DroolsTimeoutException; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; -import feign.FeignException; -import org.apache.commons.lang3.StringUtils; -import org.kie.api.KieServices; -import org.kie.api.builder.KieBuilder; -import org.kie.api.builder.KieFileSystem; -import org.kie.api.builder.ReleaseId; import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieSession; import org.kie.api.runtime.rule.QueryResults; import org.kie.api.runtime.rule.QueryResultsRow; -import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import com.google.common.util.concurrent.SimpleTimeLimiter; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; -import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.redaction.model.KieWrapper; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.knecon.fforesight.tenantcommons.TenantContext; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -47,14 +36,10 @@ import lombok.extern.slf4j.Slf4j; @Service @RequiredArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) -public class DroolsExecutionService { - - RulesClient rulesClient; +public class EntityDroolsExecutionService { EntityEnrichmentService entityEnrichmentService; - DroolsSyntaxValidationFactory droolsSyntaxValidationFactory; - RedactionServiceSettings settings; @Timed("redactmanager_executeRules") @@ -115,7 +100,7 @@ public class DroolsExecutionService { } catch (TimeoutException e) { kieSession.dispose(); - throw new DroolsTimeoutException(e, false); + throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY); } catch (InterruptedException e) { kieSession.dispose(); throw new RuntimeException(e); @@ -138,73 +123,4 @@ public class DroolsExecutionService { return fileAttributes; } - - public KieWrapper getLatestKieContainer(String dossierTemplateId) { - - long version = -1; - try { - version = rulesClient.getVersion(dossierTemplateId); - } catch (FeignException fe) { - if (fe.status() == HttpStatus.UNPROCESSABLE_ENTITY.value()) { - throw new DroolsTimeoutException(fe.getCause(), true); - } - } - - return new KieWrapper(getKieContainer(dossierTemplateId, version), version); - } - - - private ReleaseId getReleaseId(String dossierTemplate, long version) { - - KieServices kieServices = KieServices.Factory.get(); - return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate, String.format("1.%d", version)); - } - - - private KieContainer getKieContainer(String dossierTemplateId, long version) { - - KieServices kieServices = KieServices.Factory.get(); - try { - return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version)); - } catch (Exception e) { - - registerNewKieContainerVersion(dossierTemplateId, version); - return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version)); - } - - } - - - private void registerNewKieContainerVersion(String dossierTemplateId, long version) { - - var rules = rulesClient.getRules(dossierTemplateId); - if (rules == null || StringUtils.isEmpty(rules.getValue())) { - throw new RuntimeException("Rules cannot be empty."); - } - - registerNewKieContainerVersion(dossierTemplateId, version, rules.getValue()); - - } - - - private KieBuilder registerNewKieContainerVersion(String dossierTemplateId, long version, String rules) { - - KieServices kieServices = KieServices.Factory.get(); - KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); - kieFileSystem.generateAndWritePomXML(getReleaseId(dossierTemplateId, version)); - InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8)); - kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources().newInputStreamResource(input)); - KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); - return kieBuilder.buildAll(); - } - - - public DroolsSyntaxValidation testRules(String rules) { - - var versionId = System.currentTimeMillis(); - var testRules = "test-rules"; - KieBuilder kieBuilder = registerNewKieContainerVersion(testRules, versionId, rules); - return droolsSyntaxValidationFactory.buildDroolsSyntaxValidation(kieBuilder); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java new file mode 100644 index 00000000..d4adf872 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java @@ -0,0 +1,97 @@ +package com.iqser.red.service.redaction.v1.server.service.drools; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.lang3.StringUtils; +import org.kie.api.KieServices; +import org.kie.api.builder.KieBuilder; +import org.kie.api.builder.KieFileSystem; +import org.kie.api.builder.ReleaseId; +import org.kie.api.runtime.KieContainer; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.model.KieWrapper; +import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import feign.FeignException; +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class KieContainerCreationService { + + private final RulesClient rulesClient; + + + public KieWrapper getLatestKieContainer(String dossierTemplateId, RuleFileType ruleFileType) { + + long version = -1; + try { + version = rulesClient.getVersion(dossierTemplateId, ruleFileType); + } catch (FeignException fe) { + if (fe.status() == HttpStatus.UNPROCESSABLE_ENTITY.value()) { + throw new DroolsTimeoutException(fe.getCause(), true, ruleFileType); + } + if (fe.status() == HttpStatus.NOT_FOUND.value()) { + return new KieWrapper(null, -1); + } + } + if (version == -1) { + return new KieWrapper(null, version); + } + + return new KieWrapper(getKieContainer(dossierTemplateId, version, ruleFileType), version); + + } + + + private ReleaseId getReleaseId(String dossierTemplate, long version, RuleFileType ruleFileType) { + + KieServices kieServices = KieServices.Factory.get(); + return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate + ":" + ruleFileType.name(), String.format("1.%d", version)); + } + + + public KieContainer getKieContainer(String dossierTemplateId, long version, RuleFileType ruleFileType) { + + KieServices kieServices = KieServices.Factory.get(); + try { + return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version, ruleFileType)); + } catch (Exception e) { + + registerNewKieContainerVersion(dossierTemplateId, version, ruleFileType); + return kieServices.newKieContainer(getReleaseId(dossierTemplateId, version, ruleFileType)); + } + + } + + + private void registerNewKieContainerVersion(String dossierTemplateId, long version, RuleFileType ruleFileType) { + + var rules = rulesClient.getRules(dossierTemplateId, ruleFileType); + if (rules == null || StringUtils.isEmpty(rules.getValue())) { + throw new IllegalArgumentException(ruleFileType.name() + " rules cannot be empty."); + } + + registerNewKieContainerVersion(dossierTemplateId, version, rules.getValue(), ruleFileType); + } + + + public KieBuilder registerNewKieContainerVersion(String dossierTemplateId, long version, String rules, RuleFileType ruleFileType) { + + KieServices kieServices = KieServices.Factory.get(); + KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); + kieFileSystem.generateAndWritePomXML(getReleaseId(dossierTemplateId, version, ruleFileType)); + InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8)); + kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources().newInputStreamResource(input)); + KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); + return kieBuilder.buildAll(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java new file mode 100644 index 00000000..5eac41a3 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java @@ -0,0 +1,126 @@ +package com.iqser.red.service.redaction.v1.server.service.drools; + +import static java.util.stream.Collectors.groupingBy; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.drools.drl.ast.descr.GlobalDescr; +import org.drools.drl.ast.descr.ImportDescr; +import org.drools.drl.ast.descr.PackageDescr; +import org.drools.drl.ast.descr.RuleDescr; +import org.drools.drl.parser.DrlParser; +import org.kie.internal.builder.conf.LanguageLevelOption; + +import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery; +import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleType; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit; + +import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class RuleFileParser { + + private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile( + "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\"([a-zA-Z0-9]+.\\d+.\\d+)\",.*(?:, .*)?\\)"); + + + @SneakyThrows + public RuleFileBluePrint buildBluePrintFromRulesString(String ruleString) { + + DroolsSyntaxValidation customDroolsSyntaxValidation = DroolsSyntaxValidation.builder().build(); + DrlParser parser = new DrlParser(LanguageLevelOption.DRL6); + PackageDescr packageDescr = parser.parse(false, ruleString); + List allRules = new LinkedList<>(); + List allQueries = new LinkedList<>(); + + for (RuleDescr rule : packageDescr.getRules()) { + if (rule.isQuery()) { + allQueries.add(new BasicQuery(rule.getName(), rule.getLine(), ruleString.substring(rule.getStartCharacter(), rule.getEndCharacter()))); + } else { + validateRule(ruleString, rule, customDroolsSyntaxValidation, allRules); + } + } + + String imports = ruleString.substring(0, packageDescr.getImports().stream().mapToInt(ImportDescr::getEndCharacter).max().orElseThrow() + 1); + String globals = packageDescr.getGlobals() + .stream() + .map(globalDescr -> ruleString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter())) + .collect(Collectors.joining("\n")); + + List ruleClasses = buildRuleClasses(allRules); + + return new RuleFileBluePrint(imports.trim(), + packageDescr.getImports().stream().findFirst().map(ImportDescr::getLine).orElse(0), + globals.trim(), + packageDescr.getGlobals().stream().findFirst().map(GlobalDescr::getLine).orElse(0), allQueries, + ruleClasses, + customDroolsSyntaxValidation); + } + + + private static void validateRule(String ruleString, RuleDescr rule, DroolsSyntaxValidation customDroolsSyntaxValidation, List allRules) { + + BasicRule basicRule; + try { + basicRule = BasicRule.fromRuleDescr(rule, ruleString); + } catch (Exception e) { + customDroolsSyntaxValidation.addErrorMessage(rule.getLine(), rule.getColumn(), "Malformed rule name, correct format is \"\\w+.\\d+.\\d+: \""); + return; + } + if (allRules.contains(basicRule)) { + addDuplicateRuleIdentifierErrorMessage(rule, basicRule, customDroolsSyntaxValidation); + } + validateRuleIdentifierInCodeIsSame(basicRule.getCode(), basicRule.getIdentifier().toString(), rule.getLine(), customDroolsSyntaxValidation); + allRules.add(BasicRule.fromRuleDescr(rule, ruleString)); + } + + + private static void validateRuleIdentifierInCodeIsSame(String code, String identifier, int lineOffset, DroolsSyntaxValidation customDroolsSyntaxValidation) { + + Matcher matcher = ruleIdentifierInCodeFinder.matcher(code); + while (matcher.find()) { + String identifierInCode = code.substring(matcher.start(1), matcher.end(1)); + long line = code.substring(0, matcher.start(1)).lines().count() + lineOffset - 1; + if (!identifier.equals(identifierInCode)) { + customDroolsSyntaxValidation.addErrorMessage((int) line, + 0, + String.format("Rule identifier %s is not equal to rule identifier %s in rule name!", identifierInCode, identifier)); + } + } + } + + + private void addDuplicateRuleIdentifierErrorMessage(RuleDescr rule, BasicRule basicRule, DroolsSyntaxValidation customDroolsSyntaxValidation) { + + customDroolsSyntaxValidation.addErrorMessage(rule.getLine(), + rule.getColumn(), + String.format("RuleIdentifier: %s is a duplicate, duplicates are not allowed!", basicRule.getIdentifier())); + } + + + private List buildRuleClasses(List allRules) { + + List ruleTypeOrder = allRules.stream().map(BasicRule::getIdentifier).map(RuleIdentifier::type).distinct().toList(); + Map> rulesPerType = allRules.stream().collect(groupingBy(rule -> rule.getIdentifier().type())); + return ruleTypeOrder.stream().map(type -> new RuleClass(type, groupingByGroup(rulesPerType.get(type)))).collect(Collectors.toList()); + } + + + private List groupingByGroup(List rules) { + + Map> rulesPerUnit = rules.stream().collect(groupingBy(rule -> rule.getIdentifier().unit())); + return rulesPerUnit.keySet().stream().sorted().map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))).collect(Collectors.toList()); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index c4159844..8f4ddbd1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -6,13 +6,15 @@ import java.io.InputStream; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; -import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; -import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; +import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentData; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; @@ -45,6 +47,16 @@ public class RedactionStorageService { } + @SneakyThrows + public File getStoredObjectFile(String storageId) { + + File tempFile = File.createTempFile("temp", "data"); + tempFile.deleteOnExit(); + storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile); + return tempFile; + } + + @SneakyThrows public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) { @@ -86,6 +98,18 @@ public class RedactionStorageService { } + @Timed("redactmanager_getRedactionLog") + public EntityLog getEntityLog(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class); + } catch (StorageObjectDoesNotExist e) { + log.debug("RedactionLog not available."); + return null; + } + + } + @Timed("redactmanager_getDocumentGraph") public DocumentData getDocumentData(String dossierId, String fileId) { @@ -126,6 +150,16 @@ public class RedactionStorageService { } + public ComponentLog getComponentLog(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.COMPONENT_LOG), ComponentLog.class); + } catch (StorageObjectDoesNotExist e) { + throw new NotFoundException("Component Log is not available."); + } + } + + @RequiredArgsConstructor public enum StorageType { PARSED_DOCUMENT(".json"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java new file mode 100644 index 00000000..a4d5e00e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.redaction.v1.server.storage; + +import java.io.InputStream; +import java.nio.file.Path; + +import org.springframework.core.io.ClassPathResource; + +import lombok.SneakyThrows; + +public class RuleManagementResources { + + private static final String folderPrefix = "drools"; + + @SneakyThrows + public static InputStream getBaseRuleFileInputStream() { + + return new ClassPathResource(Path.of(folderPrefix, "base_rules.drl").toString()).getInputStream(); + } + + + @SneakyThrows + public static String getBaseRuleFileString() { + + try (var in = getBaseRuleFileInputStream()) { + return new String(in.readAllBytes()); + } + } + + @SneakyThrows + public static InputStream getBaseComponentRuleFileInputStream() { + + return new ClassPathResource(Path.of(folderPrefix, "base_component_rules.drl").toString()).getInputStream(); + } + + + @SneakyThrows + public static String getBaseComponentRuleFileString() { + + try (var in = getBaseComponentRuleFileInputStream()) { + return new String(in.readAllBytes()); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java new file mode 100644 index 00000000..99847d07 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java @@ -0,0 +1,49 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Locale; + +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@UtilityClass +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DateConverter { + + static List formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH), + new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH), + new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH), + new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH), + new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH), + new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH)); + + + public String convertDate(String dateAsString, String resultFormat) { + + DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH); + Date date = null; + for (SimpleDateFormat format : formats) { + + try { + date = format.parse(dateAsString); + break; + } catch (Exception e) { + log.warn("Failed to parse date from string {}. \n{}", dateAsString, e.getMessage()); + // ignore, try next... + } + } + + if (date == null) { + return dateAsString; + } + + return resultDateFormat.format(date); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/IdBuilder.java similarity index 91% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/IdBuilder.java index 7547c4a5..61b0d430 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/IdBuilder.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.utils; +package com.iqser.red.service.redaction.v1.server.utils; import java.awt.geom.Rectangle2D; import java.nio.charset.StandardCharsets; @@ -9,7 +9,7 @@ import java.util.stream.Collectors; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import lombok.experimental.UtilityClass; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/Patterns.java similarity index 95% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/Patterns.java index 74ab05bf..8d37ef67 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/Patterns.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.utils; +package com.iqser.red.service.redaction.v1.server.utils; import java.util.HashMap; import java.util.Map; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/utils/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/utils/RectangleTransformations.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java index 586e0af4..b0880c33 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/utils/RectangleTransformations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.utils; +package com.iqser.red.service.redaction.v1.server.utils; import java.awt.geom.Rectangle2D; import java.awt.geom.RectangularShape; @@ -13,7 +13,7 @@ import java.util.function.Supplier; import java.util.stream.Collector; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/utils/RedactionSearchUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java similarity index 76% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/utils/RedactionSearchUtility.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java index a43b5d43..ba82722e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/utils/RedactionSearchUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.utils; +package com.iqser.red.service.redaction.v1.server.utils; import static java.lang.String.format; @@ -9,10 +9,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.IntStream; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.experimental.UtilityClass; @@ -51,7 +50,7 @@ public class RedactionSearchUtility { } - public static TextRange findFirstBoundary(String regexPattern, CharSequence searchText) { + public static TextRange findFirstTextRange(String regexPattern, CharSequence searchText) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); Matcher matcher = pattern.matcher(searchText); @@ -66,7 +65,7 @@ public class RedactionSearchUtility { int expandedEnd; if (anyMatch(entity.getTextAfter(), regexPattern)) { - TextRange postfixTextRange = findFirstBoundary(regexPattern, entity.getTextAfter()); + TextRange postfixTextRange = findFirstTextRange(regexPattern, entity.getTextAfter()); expandedEnd = postfixTextRange.end() + entity.getTextRange().end(); } else { expandedEnd = entity.getTextRange().end(); @@ -79,7 +78,7 @@ public class RedactionSearchUtility { int expandedStart; if (anyMatch(entity.getTextBefore(), regexPattern)) { - TextRange prefixTextRange = findFirstBoundary(regexPattern, entity.getTextBefore()); + TextRange prefixTextRange = findFirstTextRange(regexPattern, entity.getTextBefore()); expandedStart = prefixTextRange.start() + entity.getTextRange().start() - entity.getTextBefore().length(); } else { expandedStart = entity.getTextRange().start(); @@ -87,7 +86,8 @@ public class RedactionSearchUtility { return expandedStart; } - public static TextRange findBoundaryOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) { + + public static TextRange findTextRangesOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) { List lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed().map(textBlock::getLineTextRange).filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary)).toList(); if (lineBoundaries.isEmpty()) { @@ -104,43 +104,43 @@ public class RedactionSearchUtility { } - public static List findBoundariesByRegex(String regexPattern, TextBlock textBlock) { + public static List findTextRangesByRegex(String regexPattern, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); - return getBoundariesByPattern(textBlock, 0, pattern); + return getTextRangesByPattern(textBlock, 0, pattern); } - public static List findBoundariesByRegex(String regexPattern, int group, TextBlock textBlock) { + public static List findTextRangesByRegex(String regexPattern, int group, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); - return getBoundariesByPattern(textBlock, group, pattern); + return getTextRangesByPattern(textBlock, group, pattern); } - public static List findBoundariesByRegexWithLineBreaks(String regexPattern, int group, TextBlock textBlock) { + public static List findTextRangesByRegexWithLineBreaks(String regexPattern, int group, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledMultilinePattern(regexPattern, false); - return getBoundariesByPatternWithLineBreaks(textBlock, group, pattern); + return getTextRangesByPatternWithLineBreaks(textBlock, group, pattern); } - public static List findBoundariesByRegexWithLineBreaksIgnoreCase(String regexPattern, int group, TextBlock textBlock) { + public static List findTextRangesByRegexWithLineBreaksIgnoreCase(String regexPattern, int group, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledMultilinePattern(regexPattern, true); - return getBoundariesByPatternWithLineBreaks(textBlock, group, pattern); + return getTextRangesByPatternWithLineBreaks(textBlock, group, pattern); } - public static List findBoundariesByRegexIgnoreCase(String regexPattern, int group, TextBlock textBlock) { + public static List findTextRangesByRegexIgnoreCase(String regexPattern, int group, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, true); - return getBoundariesByPattern(textBlock, group, pattern); + return getTextRangesByPattern(textBlock, group, pattern); } - private static List getBoundariesByPattern(TextBlock textBlock, int group, Pattern pattern) { + private static List getTextRangesByPattern(TextBlock textBlock, int group, Pattern pattern) { Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getTextRange())); List boundaries = new LinkedList<>(); @@ -151,7 +151,7 @@ public class RedactionSearchUtility { } - private static List getBoundariesByPatternWithLineBreaks(TextBlock textBlock, int group, Pattern pattern) { + private static List getTextRangesByPatternWithLineBreaks(TextBlock textBlock, int group, Pattern pattern) { String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks(); Matcher matcher = pattern.matcher(searchTextWithLineBreaks); @@ -163,7 +163,7 @@ public class RedactionSearchUtility { } - public static List findBoundariesByString(String searchString, TextBlock textBlock) { + public static List findTextRangesByString(String searchString, TextBlock textBlock) { List boundaries = new LinkedList<>(); for (int index = textBlock.indexOf(searchString); index >= 0; index = textBlock.indexOf(searchString, index + 1)) { @@ -173,10 +173,10 @@ public class RedactionSearchUtility { } - public static List findBoundariesByStringIgnoreCase(String searchString, TextBlock textBlock) { + public static List findTextRangesByStringIgnoreCase(String searchString, TextBlock textBlock) { Pattern pattern = Pattern.compile(Pattern.quote(searchString), Pattern.CASE_INSENSITIVE); - return getBoundariesByPattern(textBlock, 0, pattern); + return getTextRangesByPattern(textBlock, 0, pattern); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ResourceLoader.java similarity index 93% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ResourceLoader.java index 893381ac..2cf7258b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/ResourceLoader.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ResourceLoader.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.utils; +package com.iqser.red.service.redaction.v1.server.utils; import java.io.BufferedReader; import java.io.IOException; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/SeparatorUtils.java similarity index 92% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/SeparatorUtils.java index c872bbe2..5145876b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/SeparatorUtils.java @@ -1,10 +1,10 @@ -package com.iqser.red.service.redaction.v1.server.redaction.utils; +package com.iqser.red.service.redaction.v1.server.utils; import java.util.Set; import java.util.regex.Pattern; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/TextNormalizationUtilities.java similarity index 90% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/TextNormalizationUtilities.java index 8760fe15..fcaa358c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/TextNormalizationUtilities.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.utils; +package com.iqser.red.service.redaction.v1.server.utils; import lombok.experimental.UtilityClass; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java new file mode 100644 index 00000000..e4ee9642 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/DroolsTimeoutException.java @@ -0,0 +1,32 @@ +package com.iqser.red.service.redaction.v1.server.utils.exception; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; + +import lombok.Data; + +@Data +public class DroolsTimeoutException extends RuntimeException { + + private static final String DROOLS_TIMEOUT_MESSAGE = "Timeout during rules execution. Possible endless loop?"; + + private boolean reported; + + private RuleFileType ruleFileType; + + + public DroolsTimeoutException(Throwable cause, boolean reported, RuleFileType ruleFileType) { + + super(DROOLS_TIMEOUT_MESSAGE, cause); + this.reported = reported; + this.ruleFileType = ruleFileType; + } + + + public DroolsTimeoutException(boolean reported, RuleFileType ruleFileType) { + + super(DROOLS_TIMEOUT_MESSAGE); + this.reported = reported; + this.ruleFileType = ruleFileType; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/NotFoundException.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/NotFoundException.java similarity index 67% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/NotFoundException.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/NotFoundException.java index a25e7db2..639682f8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/NotFoundException.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/NotFoundException.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.exception; +package com.iqser.red.service.redaction.v1.server.utils.exception; public class NotFoundException extends RuntimeException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/RedactionException.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/RedactionException.java similarity index 79% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/RedactionException.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/RedactionException.java index fd801849..85170841 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/RedactionException.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/RedactionException.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.exception; +package com.iqser.red.service.redaction.v1.server.utils.exception; public class RedactionException extends RuntimeException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/RulesValidationException.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/RulesValidationException.java similarity index 71% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/RulesValidationException.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/RulesValidationException.java index dfb328da..94df763c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/exception/RulesValidationException.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/exception/RulesValidationException.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.exception; +package com.iqser.red.service.redaction.v1.server.utils.exception; public class RulesValidationException extends RuntimeException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl new file mode 100644 index 00000000..d55381be --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl @@ -0,0 +1,43 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; + +global ComponentCreationService componentCreationService + +/** +The imports, globals, queries and rules from this file are required for any component rule file. +Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution! +*/ + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +query "getComponents" + $component: Component() + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl new file mode 100644 index 00000000..17205da4 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl @@ -0,0 +1,87 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; + +global Document document +global EntityCreationService entityCreationService +global ManualChangesApplicationService manualChangesApplicationService +global Dictionary dictionary + +/** +The imports, globals, queries and rules from this file are required for any entity rule file. +Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution! +*/ + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +//------------------------------------ Local dictionary search rules ------------------------------------ + +// Rule unit: LocalDictionarySearch.0 +rule "LDS.0.0: run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) + .forEach(entity -> { + Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); + entity.addMatchedRules(matchedRules); + }); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 787d6481..520914d5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -11,9 +11,9 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; -import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; -import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; -import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; +import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; +import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader; +import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; import com.iqser.red.storage.commons.service.StorageService; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index f2eac69f..7c0c5abe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -33,8 +33,8 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantsClient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 9b356a89..cd0b91c1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.io.FileOutputStream; @@ -23,11 +24,13 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; @@ -40,6 +43,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext; public class DocumineFloraTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/documine_flora.drl"); + private static final String COMPONENT_RULES = loadFromClassPath("drools/documine_flora_components.drl"); @Test @@ -143,8 +147,10 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES)); loadTypeForTest(); loadNerForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index be263ca9..878a9fd8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import static org.wildfly.common.Assert.assertTrue; @@ -26,6 +27,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; @@ -36,6 +38,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; @@ -70,8 +73,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); loadDictionaryForTest(); loadTypeForTest(); @@ -165,7 +169,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { System.out.println("Finished analysis"); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> !e.isRecommendation()).filter(e -> e.getMatchedRule().startsWith("CBI.16")) + RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> !e.isRecommendation()) + .filter(e -> e.getMatchedRule().startsWith("CBI.16")) .findAny() .orElseThrow(); IdRemoval removal = buildIdRemoval(desireeEtAl.getId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 034969be..97e1ec1d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.io.FileOutputStream; @@ -43,14 +44,15 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; @@ -61,11 +63,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; -import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; @@ -102,8 +105,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); loadDictionaryForTest(); loadTypeForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 77602279..186ceb78 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -1,11 +1,13 @@ package com.iqser.red.service.redaction.v1.server; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.util.List; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -20,10 +22,11 @@ import org.springframework.context.annotation.Primary; import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; @@ -32,6 +35,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.SneakyThrows; +@Disabled // TODO: re-enable, when redaction log is removed @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class) @@ -60,8 +64,9 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); loadDictionaryForTest(); loadTypeForTest(); @@ -132,7 +137,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); - assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); + assertThat(redactionLogEntry.getEngines().contains(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine.DICTIONARY)).isEqualTo(true); } @@ -170,7 +175,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); - assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); + assertThat(redactionLogEntry.getEngines().contains(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine.DICTIONARY)).isEqualTo(true); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 2ed07326..0cb8b29a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -1,34 +1,33 @@ package com.iqser.red.service.redaction.v1.server; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*; -import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; -import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; -import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.controller.RedactionController; -import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; -import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; -import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; -import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; -import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; -import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; -import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; -import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; -import com.knecon.fforesight.tenantcommons.TenantContext; -import com.knecon.fforesight.tenantcommons.TenantsClient; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; +import static java.util.Map.entry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -47,26 +46,56 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.*; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import java.io.*; -import java.math.BigInteger; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.DirectoryStream; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.MessageDigest; -import java.time.OffsetDateTime; -import java.util.*; -import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; +import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.controller.RedactionController; +import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; +import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; +import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader; +import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; +import com.knecon.fforesight.tenantcommons.TenantContext; +import com.knecon.fforesight.tenantcommons.TenantsClient; -import static java.util.Map.entry; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; @Slf4j @ExtendWith(SpringExtension.class) @@ -233,8 +262,9 @@ public class RulesTest { objectMapper.registerModule(new JavaTimeModule()); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); loadDictionaryForTest(); loadTypeForTest(); @@ -401,24 +431,23 @@ public class RulesTest { analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER); analyzeService.analyze(request); - RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); // All timestamps are ignored, because they are for sure different - assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion()); - assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber()); - assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion()); - assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion()); - assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion()); - assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion()); + assertThat(entityLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion()); + assertThat(entityLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber()); + assertThat(entityLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion()); + assertThat(entityLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion()); + assertThat(entityLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion()); + assertThat(entityLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion()); - assertThat(redactionLog.getRedactionLogEntry() - .stream() - .filter(r -> !r.isFalsePositive()) + assertThat(entityLog.getEntityLogEntry() + .stream().filter(r -> !r.getEntryType().equals(EntryType.FALSE_POSITIVE)).filter(r -> !r.getEntryType().equals(EntryType.FALSE_RECOMMENDATION)) .collect(Collectors.toSet()) .size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size()); - assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size()); + assertThat(entityLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size()); - for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) { + for (EntityLogLegalBasis redactionLegalBasis : entityLog.getLegalBasis()) { var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis() .stream() .filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName())) @@ -428,7 +457,7 @@ public class RulesTest { assertThat(savedRedactionLegalBasis).isPresent(); } - for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) { + for (EntityLogEntry redactionLogEntry : entityLog.getEntityLogEntry()) { var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst(); assertThat(savedRedactionLogEntry).isPresent(); assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId()); @@ -436,13 +465,12 @@ public class RulesTest { assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue()); assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason()); assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule()); - assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle()); assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis()); assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported()); - assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted()); - assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint()); - assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation()); - assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive()); + assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.getState().equals(EntryState.APPLIED)); + assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.HINT)); + assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.RECOMMENDATION)); + assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE)); assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection()); assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor()); assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber()); @@ -450,43 +478,30 @@ public class RulesTest { assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter()); assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset()); assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset()); - assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage()); + assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.IMAGE)); assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency()); assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry()); assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry()); assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded()); - assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId()); - for (Rectangle rectangle : redactionLogEntry.getPositions()) { + for (Position position : redactionLogEntry.getPositions()) { var savedRectangle = savedRedactionLogEntry.get() .getPositions() .stream() - .filter(r -> r.getPage() == rectangle.getPage()) - .filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX()) - .filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY()) - .filter(r -> r.getHeight() == rectangle.getHeight()) - .filter(r -> r.getWidth() == rectangle.getWidth()) + .filter(r -> r.getPage() == position.getPageNumber()) + .filter(r -> r.getTopLeft().getX() == position.x()) + .filter(r -> r.getTopLeft().getY() == position.y()) + .filter(r -> r.getHeight() == position.w()) + .filter(r -> r.getWidth() == position.h()) .findFirst(); assertThat(savedRectangle).isPresent(); } - for (RedactionLogComment comment : redactionLogEntry.getComments()) { - var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst(); - assertThat(savedComment).isPresent(); - assertThat(savedComment.get().getId()).isEqualTo(comment.getId()); - assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser()); - assertThat(savedComment.get().getText()).isEqualTo(comment.getText()); - assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId()); - assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId()); - - } - for (Change change : redactionLogEntry.getChanges()) { var savedChange = savedRedactionLogEntry.get() .getChanges() .stream() - .filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber()) - .filter(c -> c.getType() == change.getType()) + .filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber()).filter(c -> c.getType().name().equals(change.getType().name())) .findFirst(); assertThat(savedChange).isPresent(); } @@ -496,14 +511,14 @@ public class RulesTest { .getManualChanges() .stream() .filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus()) - .filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType()) + .filter(m -> m.getManualRedactionType().name().equals(manualChange.getManualRedactionType().name())) .filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId())) .filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges()) .findFirst(); assertThat(savedManualChange).isPresent(); } - assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(new Engine[0])); + assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(Engine[]::new)); assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference()); assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java index 54dee8cc..98875593 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java @@ -13,7 +13,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.pdfbox.Loader; -import org.apache.pdfbox.io.MemoryUsageSetting; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.PDPageContentStream; @@ -36,10 +35,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle; -import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations; -import com.iqser.red.service.redaction.v1.server.exception.RedactionException; -import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException; import lombok.RequiredArgsConstructor; @@ -67,14 +66,14 @@ public class AnnotationService { */ public AnnotateResponse annotate(AnnotateRequest annotateRequest) { - var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), + var storedObjectFile = redactionStorageService.getStoredObjectFile(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN)); var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId()); var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId()); - try (PDDocument pdDocument = Loader.loadPDF(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) { + try (PDDocument pdDocument = Loader.loadPDF(storedObjectFile)) { pdDocument.setAllSecurityToBeRemoved(true); dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java index f594a1a3..c6e30b9a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java @@ -5,9 +5,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import org.junit.jupiter.api.Test; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; public class TextEntityTest { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java index 07dea061..26f4b8a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java @@ -16,8 +16,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest; import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService; -import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java similarity index 92% rename from redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java index d35cc3c8..38a52644 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java @@ -16,21 +16,23 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.springframework.beans.factory.annotation.Autowired; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Headline; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; -public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest { +public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 7dcf7398..be3f9132 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph; -import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; +import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.awt.Color; @@ -29,25 +30,27 @@ import org.springframework.context.annotation.Import; import org.springframework.core.io.ClassPathResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; -import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.SneakyThrows; @@ -65,7 +68,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration private EntityCreationService entityCreationService; @Autowired - private DroolsExecutionService droolsExecutionService; + private EntityDroolsExecutionService entityDroolsExecutionService; @Qualifier("kieContainer") @Autowired @@ -98,8 +101,9 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration entityCreationService = new EntityCreationService(entityEnrichmentService); TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); loadDictionaryForTest(); loadTypeForTest(); @@ -151,7 +155,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart); long droolsStart = System.currentTimeMillis(); - List fileAttributes = droolsExecutionService.executeRules(kieContainer, + List fileAttributes = entityDroolsExecutionService.executeRules(kieContainer, document, dictionary, Collections.emptyList(), @@ -251,7 +255,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration var tmpFileName = "/tmp/" + filename.split("/")[2] + "_ENTITY_BBOX.pdf"; var fileResource = new ClassPathResource(filename + ".pdf"); - try (var fileStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(fileStream)) { + try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) { for (Page page : document.getPages()) { List entityPositionsOnPage = page.getEntities() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTableIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTableIntegrationTest.java index 675b92e3..cded6d0f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTableIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentTableIntegrationTest.java @@ -4,10 +4,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.Test; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; public class DocumentTableIntegrationTest extends BuildDocumentIntegrationTest { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java index 84e870cb..024660e8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java @@ -9,8 +9,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.springframework.core.io.ClassPathResource; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; import lombok.SneakyThrows; @@ -79,7 +79,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati var tmpFileName = "/tmp/" + filename.split("/")[2] + "_SEMANTIC_NODES_BBOX.pdf"; var fileResource = new ClassPathResource(filename + ".pdf"); - try (var fileStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(fileStream)) { + try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) { PdfVisualisationUtility.drawDocumentGraph(pdDocument, document); PdfVisualisationUtility.drawTextBlock(pdDocument, diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java index 60c9d715..0f5a8c77 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph; import static java.util.stream.Collectors.toMap; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.io.IOException; @@ -21,15 +22,17 @@ import com.google.common.collect.Sets; import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.CustomEntityCreationAdapter; -import com.iqser.red.service.redaction.v1.server.redaction.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; +import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -40,7 +43,7 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest { private static final String RULES = loadFromClassPath("drools/rules.drl"); @Autowired - private CustomEntityCreationAdapter redactionLogAdapter; + private ManualEntityCreationService redactionLogAdapter; @Autowired private RedactionLogCreatorService redactionLogCreatorService; @@ -49,8 +52,10 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest { public void stubClients() { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); + loadDictionaryForTest(); loadTypeForTest(); loadNerForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java index 1d9499d0..8fc4834f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java @@ -7,12 +7,12 @@ import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; public class SearchImplementationTest extends BuildDocumentIntegrationTest { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TextRangeTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TextRangeTest.java index 9c464d43..32d678d5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TextRangeTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TextRangeTest.java @@ -11,6 +11,8 @@ import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; + class TextRangeTest { TextRange startTextRange; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java new file mode 100644 index 00000000..c3b017bc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java @@ -0,0 +1,245 @@ +package com.iqser.red.service.redaction.v1.server.drools.files.management.services; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockitoAnnotations; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.RuleValidationModel; +import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService; +import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; +import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; +import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; + +import lombok.SneakyThrows; + +class DroolsSyntaxValidationServiceTest { + + @MockBean + RulesClient rulesClient; + @MockBean + EntityEnrichmentService entityEnrichmentService; + + + @BeforeEach + public void setupMocks() { + + MockitoAnnotations.openMocks(this); + } + + + @Test + @SneakyThrows + void testRules() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertTrue(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testAllRules() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/all_rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertTrue(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testAcceptanceRules() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/acceptance_rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertTrue(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testManualRedactionRules() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertTrue(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testRulesV2() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/rules_v2.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertTrue(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testDocumineRules() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/documine_flora.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertTrue(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testCorruptedRules() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + String corruptedRules = rulesString.replaceAll(";", ""); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); + assertFalse(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testCorruptedImports() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;", ""); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); + assertFalse(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testRemoveQuery() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/rules.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + String corruptedRules = rulesString.replaceAll("query \"getFileAttributes\"\n.*\n *end", ""); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertFalse(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testComponentDrools() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/documine_flora_components.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.COMPONENT.name(), rulesString)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertTrue(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void testComponentDroolsAsEntityDrools() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + var rulesFile = new ClassPathResource("drools/documine_flora_components.drl"); + + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); + assertFalse(droolsSyntaxValidation.isCompiled()); + } + + + @Test + @SneakyThrows + void attemptImportsFixToAllRuleFiles() { + + DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient)); + + List ruleFiles = List.of("drools/rules.drl", + "drools/all_rules.drl", + "drools/documine_flora.drl", + "drools/manual_redaction_rules.drl", + "drools/acceptance_rules.drl", + "drools/rules_v2.drl"); + + for (String ruleFile : ruleFiles) { + var rulesFile = new ClassPathResource(ruleFile); + String rulesString = new String(rulesFile.getInputStream().readAllBytes()); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString)); + if (droolsSyntaxValidation.isCompiled()) { + continue; + } + RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString); + RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString()); + + rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports()); + rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals()); + + try (OutputStream outStream = new FileOutputStream(rulesFile.getFile().getAbsolutePath().replace("/test", "").replace("build", "src/test"))) { + outStream.write(rulesString.getBytes(StandardCharsets.UTF_8)); + } + + } + + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 2977e332..7ae07152 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.manualchanges; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import static org.wildfly.common.Assert.assertTrue; @@ -33,6 +34,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; @@ -53,13 +55,14 @@ import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; -import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; @@ -107,8 +110,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { TenantContext.setTenantId("redaction"); - when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(System.currentTimeMillis()); - when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES)); + doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); + loadDictionaryForTest(); loadTypeForTest(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index 6b6f04c9..629fa5c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -34,15 +34,15 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.PositionOnPage; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; @Import(ManualChangesIntegrationTest.TestConfiguration.class) public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index b5f8e0c8..21e02bd1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -17,11 +17,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/CustomEntityCreationAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java similarity index 82% rename from redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/CustomEntityCreationAdapterTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java index ce22183a..d33f2724 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/CustomEntityCreationAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java @@ -21,25 +21,25 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.CustomEntityCreationAdapter; -import com.iqser.red.service.redaction.v1.server.redaction.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; -import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService; import lombok.SneakyThrows; -public class CustomEntityCreationAdapterTest extends BuildDocumentIntegrationTest { +public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; @Autowired - private CustomEntityCreationAdapter customEntityCreationAdapter; + private ManualEntityCreationService manualEntityCreationService; @Autowired private RedactionLogCreatorService redactionLogCreatorService; @@ -88,7 +88,7 @@ public class CustomEntityCreationAdapterTest extends BuildDocumentIntegrationTes tempEntity.removeFromGraph(); assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = customEntityCreationAdapter.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document); + List notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document); assertTrue(notFoundManualEntities.isEmpty()); assertEquals(1, document.getEntities().size()); } @@ -116,7 +116,7 @@ public class CustomEntityCreationAdapterTest extends BuildDocumentIntegrationTes assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = customEntityCreationAdapter.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document); + List notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document); assertEquals(1, notFoundManualEntities.size()); assertTrue(document.getEntities().isEmpty()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java index a3c539be..08a261e2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java @@ -22,6 +22,7 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest public static final String FILE_NAME = "test-file"; + @Test @SneakyThrows public void testFile() { @@ -30,21 +31,18 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest om.registerModule(new JavaTimeModule()); var file = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".pdf").getInputStream(); - redactionStorageService.storeObject("dossierId", "fileId", FileType.ORIGIN, file); + redactionStorageService.storeObject(DOSSIER_ID, FILE_ID, FileType.ORIGIN, file); try { var nerData = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".ner.json").getInputStream(); - redactionStorageService.storeObject("dossierId", "fileId", FileType.NER_ENTITIES, nerData); + redactionStorageService.storeObject(DOSSIER_ID, FILE_ID, FileType.NER_ENTITIES, nerData); } catch (Exception e) { log.warn("No NER File Provided"); redactionServiceSettings.setNerServiceEnabled(false); } - AnalyzeRequest ar = AnalyzeRequest.builder() - .fileId("fileId") - .dossierId("dossierId") - .analysisNumber(1) - .dossierTemplateId("dossierTemplateId") + AnalyzeRequest ar = AnalyzeRequest.builder().fileId(FILE_ID).dossierId(DOSSIER_ID) + .analysisNumber(1).dossierTemplateId(DOSSIER_TEMPLATE_ID) .lastProcessed(OffsetDateTime.now()) .excludedPages(Set.of()) .fileAttributes(List.of()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java index 2b29489a..4ca322b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java @@ -1,10 +1,10 @@ package com.iqser.red.service.redaction.v1.server.realdata; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; import java.util.ArrayList; @@ -38,19 +38,21 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver; -import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -65,6 +67,10 @@ import lombok.SneakyThrows; @AutoConfigureObservability public class LiveDataIntegrationTest { + public static final String DOSSIER_ID = "dossierId"; + public static final String FILE_ID = "fileId"; + public static final String DOSSIER_TEMPLATE_ID = "dossierTemplateId"; + protected static String BASE_DIR = "performance/"; protected static String EFSA_SANITISATION_GFL_V1 = "dictionaries/EFSA_sanitisation_GFL_v1/"; @@ -132,11 +138,13 @@ public class LiveDataIntegrationTest { //when(tableServiceResponseAdapter.convertTables(anyString(), anyString())).thenReturn(new HashMap<>()); TenantContext.setTenantId("redaction"); - when(dictionaryClient.getVersion(anyString())).thenReturn(1L); - when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L); + when(dictionaryClient.getVersion(DOSSIER_TEMPLATE_ID)).thenReturn(1L); + when(dictionaryClient.getVersionForDossier(DOSSIER_ID)).thenReturn(1L); var rules = IOUtils.toString(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "rules.drl").getInputStream()); - when(rulesClient.getRules(any())).thenReturn(JSONPrimitive.of(rules)); + when(rulesClient.getRules(DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(rules)); + when(rulesClient.getRules(DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(rules)); + doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT); ObjectMapper objectMapper = new ObjectMapper(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index 95ca0e3b..c5ae5d41 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -20,15 +20,17 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.PositionOnPage; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; import lombok.SneakyThrows; @@ -69,7 +71,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.textRange().start() < entity.textRange().end())); ClassPathResource resource = new ClassPathResource(filePath); - try (PDDocument pdDocument = Loader.loadPDF(resource.getInputStream())) { + try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) { Stream unchangedAddressParts = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) .getNerEntityList() @@ -118,7 +120,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getTextRange().start() < entity.getTextRange().end())); ClassPathResource resource = new ClassPathResource(filePath); - try (PDDocument pdDocument = Loader.loadPDF(resource.getInputStream())) { + try (PDDocument pdDocument = Loader.loadPDF(resource.getFile())) { List validatedEntities = NerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), document) .getNerEntityList() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionServiceTest.java deleted file mode 100644 index 1d3a8897..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionServiceTest.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.service; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.MockitoAnnotations; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.core.io.ClassPathResource; - -import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; -import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService; - -import lombok.SneakyThrows; - -class DroolsExecutionServiceTest { - - @MockBean - RulesClient rulesClient; - @MockBean - EntityEnrichmentService entityEnrichmentService; - - - @BeforeEach - public void setupMocks() { - - MockitoAnnotations.openMocks(this); - } - - - @Test - @SneakyThrows - void testRules() { - - DroolsExecutionService droolsExecutionService = new DroolsExecutionService(rulesClient, entityEnrichmentService, new DroolsSyntaxValidationFactory(), new RedactionServiceSettings()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsExecutionService.testRules(rulesString); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - @Test - @SneakyThrows - void testAllRules() { - - DroolsExecutionService droolsExecutionService = new DroolsExecutionService(rulesClient, entityEnrichmentService, new DroolsSyntaxValidationFactory(), new RedactionServiceSettings()); - var rulesFile = new ClassPathResource("drools/all_rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsExecutionService.testRules(rulesString); - assertTrue(droolsSyntaxValidation.isCompiled()); - } - - - @Test - @SneakyThrows - void testCorruptedRules() { - - DroolsExecutionService droolsExecutionService = new DroolsExecutionService(rulesClient, entityEnrichmentService, new DroolsSyntaxValidationFactory(), new RedactionServiceSettings()); - var rulesFile = new ClassPathResource("drools/rules.drl"); - - String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - String corruptedRules = rulesString.replaceAll(";", ""); - - DroolsSyntaxValidation droolsSyntaxValidation = droolsExecutionService.testRules(corruptedRules); - assertFalse(droolsSyntaxValidation.isCompiled()); - } - -} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java index 526c9884..a17350bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java @@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities; + public class TextNormalizationUtilitiesTest { @Test diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/ExceptionProvider.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/ExceptionProvider.java new file mode 100644 index 00000000..65a5a1db --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/ExceptionProvider.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.util.Collections; + +import feign.FeignException; +import feign.Request; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class ExceptionProvider { + + public FeignException.NotFound getMockNotFoundFeignException() { + + return new FeignException.NotFound("", Request.create(Request.HttpMethod.GET, "", Collections.emptyMap(), null, null, null), null, null); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java index 30b6d20d..ef6cc46d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java @@ -21,6 +21,7 @@ public class LayoutParsingRequestProvider { var positionBlockFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION); var pageFileStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES); var simplifiedTextStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SIMPLIFIED_TEXT); + var viewerDocumentStorageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.VIEWER_DOCUMENT); return LayoutParsingRequest.builder() .layoutParsingType(layoutParsingType) .originFileStorageId(originFileStorageId) @@ -31,7 +32,7 @@ public class LayoutParsingRequestProvider { .positionBlockFileStorageId(positionBlockFileStorageId) .pageFileStorageId(pageFileStorageId) .sectionGridStorageId(sectionGridStorageId) - .simplifiedTextStorageId(simplifiedTextStorageId) + .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java index 45f4686f..85771fa3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java @@ -13,12 +13,12 @@ import org.apache.pdfbox.pdmodel.PDPageContentStream; import org.apache.pdfbox.pdmodel.font.PDType1Font; import org.apache.pdfbox.pdmodel.font.Standard14Fonts; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import lombok.AccessLevel; import lombok.Builder; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 34d47b20..ea8db469 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -1,8 +1,8 @@ package drools import static java.lang.String.format; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.anyMatch; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.exactMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; import java.util.List; import java.util.LinkedList; @@ -12,41 +12,47 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; -import com.iqser.red.service.redaction.v1.server.document.graph.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.*; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; -import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; global Document document global EntityCreationService entityCreationService diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl new file mode 100644 index 00000000..edf7fb72 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl @@ -0,0 +1,962 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.document.graph.*; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.*; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; +import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; +import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType +import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType +import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility +import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity; + +global Document document +global EntityCreationService entityCreationService +global ManualRedactionApplicationService manualRedactionApplicationService +global Dictionary dictionary + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +//--------------------------------------------------------------------------- + +rule "H.0.0 retract table of contents page" + when + $page: Page(getMainBodyTextBlock().getSearchText().contains("........") || (getMainBodyTextBlock().getSearchText().contains("APPENDICES") && getMainBodyTextBlock().getSearchText().contains("TABLES"))) + $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE) + then + retract($node); + end + + +rule "H.0.0: Ignore Table of Contents" + salience 10 + when + $tocHeadline: Headline(containsString("CONTENTS")) + $page: Page() from $tocHeadline.getParent().getPages() + $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1)) + then + retract($node); + end + + +// Rule unit: MAN.0 +/* +rule "H.0.0: Show headlines" + when + $headline: Headline() + then + entityCreationService.bySemanticNode($headline, "headline", EntityType.ENTITY); + end +*/ + +/* +rule "NER.0.0: Show NER Countries" + when + $nerEntities: NerEntities(hasEntitiesOfType("COUNTRY")) + $section: Section() + then + $nerEntities + .streamEntitiesOfType("COUNTRY") + .map( + nerEntity -> entityCreationService + .byNerEntity( + nerEntity, + "ner_country", + EntityType.ENTITY, + $section + ) + ) + .forEach( + entity -> { + entity.skip("NER.0.0", "Country found by NER"); + insert(entity); + } + ); + end + +rule "NER.0.1: Show NER Cities" + when + $nerEntities: NerEntities(hasEntitiesOfType("CITY")) + $section: Section() + then + $nerEntities + .streamEntitiesOfType("CITY") + .map( + nerEntity -> entityCreationService + .byNerEntity( + nerEntity, + "ner_city", + EntityType.ENTITY, + $section + ) + ) + .forEach( + entity -> { + entity.skip("NER.0.1", "City found by NER"); + insert(entity); + } + ); + end + */ + +/* +rule "NER.0.0: Show NER Entities" + when + $nerEntities: NerEntities( + hasEntitiesOfType("STREET"), + (hasEntitiesOfType("CARDINAL") || hasEntitiesOfType("POSTAL") || hasEntitiesOfType("CITY") || hasEntitiesOfType("COUNTRY")) + ) + then + NerEntitiesAdapter.combineNerEntities( + $nerEntities, + Set.of("STREET"), + Set.of("STREET","CARDINAL","POSTAL","CITY","COUNTRY"), + 20, + 3, + false + ) + .forEach( + nerBoundary -> entityCreationService.byBoundary( + nerBoundary, + "ner", + EntityType.ENTITY, + document + ) + .ifPresent( + entity -> entity.skip("NER.0.0","NER address.") + ) + ); + end + */ + +rule "DOC.1.0: Adama number" + when + $paragraph: Paragraph( + containsString("R-") + && onPage(1) + ) + then + entityCreationService + .byRegex( + "0?0?R\\-[l\\d]{3,5}\\p{Lu}?", + "adama_number", + EntityType.ENTITY, + $paragraph) + .findFirst() + .ifPresent(entity -> { + entity.apply("DOC.1.0", "Adama number found"); + insert(FileAttribute.builder().label("Adama").value(entity.getValue()).build()); + }); + end + +rule "DOC.2.0: Study number by keyword" + when + $studyNumberKeyword: String() from List.of( + "BioChem project number:", + "DTI Report.", + "Final Report N°", + "LPT Report No.", + "Project identity", + "Project No.:", + "Protocol No.", + "PTRL Report No.", + "SLI Report #", + "SLI Study #", + "Specht Analytical Study Plan", + "SPL PROJECT NUMBER:", + "Study code:", + "Study N°", + "Study-No.", + "Study No:", + "Study number:", + "Study Number" + ) + $excludeKeyWords: String() from List.of( + "Sponsors study number" + ) + $paragraph: Paragraph( + containsStringIgnoreCase($studyNumberKeyword) + && !containsStringIgnoreCase($excludeKeyWords) + && onPage(1) + ) + then + entityCreationService + .lineAfterStringIgnoreCase($studyNumberKeyword,"study_number",EntityType.ENTITY,$paragraph) + .findFirst() + .ifPresent(entity -> + { + entity.apply("DOC.2.0", "Study number found"); + insert(FileAttribute.builder().label("Study").value(entity.getValue()).build()); + } + ); + end + +rule "DOC.2.1: Study number in header" + when + $studyNumberKeyword: String() from List.of( + "Final Report" + ) + $header: Header( + containsStringIgnoreCase($studyNumberKeyword) + && onPage(2) + ) + then + entityCreationService + .lineAfterStringIgnoreCase($studyNumberKeyword,"study_number",EntityType.ENTITY,$header) + .findFirst() + .ifPresent(entity -> + { + entity.apply("DOC.2.1", "Study number found"); + insert(FileAttribute.builder().label("Study").value(entity.getValue()).build()); + } + ); + end + +// this rule breaks documents +/* +rule "DOC.2.2: Project Number with Section Title" + when + $title: String() from List.of( + "LABORATORY PROJECT ID." + ) + $paragraph: Paragraph( + containsStringIgnoreCase($title) + && onPage(1) + ) + then + entityCreationService.semanticNodeAfterString( + $paragraph, + "LABORATORY PROJECT ID.", + "study_number", + EntityType.ENTITY + ) + .ifPresent( + entity -> entity.apply("DOC.2.2","Study number by title found.") + ); + end + */ + + +rule "DOC.3.0: Batch Material Number" + when + $hlKeyword: String() from List.of( + "formulation", + "formulation:", + "test item", + "test substance", + "test material" + ) + $hlNoKeyword: String() from List.of( + "Preparation" + ) + $headline: Headline( + containsStringIgnoreCase($hlKeyword) + && !containsStringIgnoreCase($hlNoKeyword) + ) + $batchKeyword: String() from List.of( + "Batch number", + "Batch number:", + "Batch number* :", + "batch no", + "Batch no.", + "Lot.n.:" + ) + $section: Section( + containsStringIgnoreCase($batchKeyword) + && ( + getHeadline() == $headline + || ( + containsStringIgnoreCase($hlKeyword) + && !containsStringIgnoreCase($hlNoKeyword) + ) + ) + ) + then + entityCreationService + .lineAfterStringIgnoreCase($batchKeyword,"batch_material_number",EntityType.ENTITY,$section) + .forEach( + entity -> { + entity.apply("DOC.3.0","Batch number found."); + insert(FileAttribute.builder().label("Batch No").value(entity.getValue()).build()); + } + ); + // how can I unset the fileAttribute if there is no match any longer? + end + +rule "DOC.4.0: Study title by document structure" + when + $headline: Headline(containsStringIgnoreCase("FINAL REPORT")) + $section: Section(getHeadline() == $headline && onPage(1)) + then + $section + .streamAllSubNodesOfType(NodeType.PARAGRAPH) + .findFirst() + .ifPresent( + paragraph -> entityCreationService + .bySemanticNode( + $headline, + "title", + EntityType.ENTITY + ) + .ifPresent( + entity -> entity.apply("DOC.4.0","Report title found.") + ) + ); + end + +rule "DOC.4.1: study title as headline" + when + not Table(onPage(1)) + $hlKeyword: String() from List.of( + "Analyitical Method", + "Residues", + "Acute" + ) + $headline: Headline( + onPage(1) + && containsStringIgnoreCase($hlKeyword) + ) + then + entityCreationService.bySemanticNode($headline, "title", EntityType.ENTITY).ifPresent(entity -> + entity.apply("DOC.4.1", "Title found", "n-a") + ); + end + + +rule "DOC.5.0: Author with headline or on pages 1 and 2" + when + $hlKeyword: String() from List.of( + "sponsor", + "personnel", + "staff involved", + "study details", + "management of study" + ) + $keyword: String() from List.of( + "Study Director", + "Studv Director", + "Study Director:", + "Study Director :", + "stray Birector:", + "Author:", + "Author", + "Author(s)" + ) + $section: Section( + containsStringIgnoreCase($keyword) + && ( + getHeadline().containsStringIgnoreCase($hlKeyword) + || (onPage(1) || onPage(2)) + ) + ) + then + entityCreationService + .lineAfterStringIgnoreCase( + $keyword, + "author", + EntityType.ENTITY, + $section + ) + .findFirst() + .ifPresent( + entity -> { + entity.apply("DOC.5.0","Author found."); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + } + ); + end + +// separate rule as I cannot save a righthand side with two entityCreationService-calls +/* +rule "DOC.5.1: Author" + when + $keyword: String() from List.of( + "Author(s)", + "Author", + "Authors" + ) + $stopKeyword: String() from List.of( + "Test Facility", + "Study Initiation", + "Study completed on" + ) + $author: RedactionEntity(type=="author") + $section: Section( + containsStringIgnoreCase($keyword) + && containsStringIgnoreCase($stopKeyword) + && onPage(1) + ) + then + // betweenStringsIgnoreCase captures the longest possible string (should be the shortest?) + entityCreationService + .betweenStrings( + $keyword, + $stopKeyword, + "author", + EntityType.ENTITY, + $section + ) + .forEach( + entity -> { + entity.apply("DOC.5.1","Author found."); + insert(FileAttribute.builder().label("Author").value(entity.getValue()).build()); + } + ); + end +*/ + +rule "DOC.5.2: Authors list on cover page paragraph" + when + $sectionTitle: String() from List.of( + "Study Director" + ) + $author: RedactionEntity(type=="author", isApplied()) + $node: Paragraph( + onPage(1) + && entities not contains $author + && containsStringIgnoreCase($sectionTitle) + ) + then + entityCreationService + .byRegexWithLineBreaksIgnoreCase( + "([\\w\\(\\) .]{5,30})\\n", + "author", + EntityType.ENTITY, + 1, + $node + ) + .filter( + entity -> !entity.getValue().toLowerCase().contains($sectionTitle.toLowerCase()) + ) + .forEach( + entity -> entity.apply("DOC.5.2","Author list on cover page found") + ); + end + +rule "DOC.5.3: Authors list on cover page section" + when + $author: RedactionEntity(type=="author", isApplied()) + $sectionTitle: String() from List.of( + "Study Director" + ) + $node: Section( + onPage(1) + && entities not contains $author + && containsStringIgnoreCase($sectionTitle) + ) + then + entityCreationService + .byRegexWithLineBreaksIgnoreCase($sectionTitle + "\\n([\\w\\(\\) .]{5,30})\\n","author",EntityType.ENTITY,1,$node) + .forEach( + entity -> entity.apply("DOC.5.3","Author list on cover page found") + ); + end + +rule "DOC.5.9: Remove skipped authors" + salience 9999 + when + $author: RedactionEntity(type=="author", !isApplied()) + then + $author.removeFromGraph(); + end + +/* +rule "DOC.6.0: Finalization Date" + when + $headline: Headline( + containsStringIgnoreCase("study dates") + ) + $section: Section( + (getHeadline() == $headline) + && containsStringIgnoreCase("final report") + ) + then + entityCreationService + .lineAfterStrings( + List.of( + "final report" + ), + "finalization_date", + EntityType.ENTITY, + $section + ) + .forEach( + entity -> entity.apply("DOC.6.0", "Experimental end date found", "n-a") + ); + end + +rule "DOC.7.0: Test on vertebrates" + when + $headline: Headline( + containsStringIgnoreCase("animal") + ) + $species: RedactionEntity(type=="species") + $paragraph: Paragraph( + getHeadline() == $headline + && entities contains $species + && containsStringIgnoreCase("species") + ) + then + $species.apply("DOC.7.0","Species found in animal section"); + end + +rule "DOC.8.0: Testing Facility name using NER entity boundaries" + when + $nerEntities: NerEntities( + hasEntitiesOfType("STREET"), + (hasEntitiesOfType("CARDINAL") || hasEntitiesOfType("POSTAL") || hasEntitiesOfType("CITY") || hasEntitiesOfType("COUNTRY")) + ) + $testFacilityKeyword: String() from List.of( + "Test facility", + "Test Facility:", + "Testing Facility:" + ) + $paragraph: Paragraph( + containsStringIgnoreCase("facility") + ) + then + entityCreationService.byString( + $testFacilityKeyword, + "hint", + EntityType.ENTITY, + $paragraph + ) + .forEach( + ent -> ent.apply("DOC.8.0","Facility hint") + ); + */ + /* + // Needs BugFixing - Documents with NerEntities in $paragraph + // switch to state "Re-processing required" + entityCreationService.betweenBoundaries( + RedactionSearchUtility.findBoundariesByStringIgnoreCase( + $testFacilityKeyword, + $paragraph.getTextBlock() + ), + NerEntitiesAdapter.combineNerEntities( + $nerEntities, + Set.of("STREET"), + Set.of("STREET","CARDINAL","POSTAL","CITY","COUNTRY"), + 20, + 3, + false + ).toList(), + "testing_facility_name", + EntityType.ENTITY, + $paragraph + ) + .forEach( + entity -> entity.apply("DOC.8.0","Test Facility name found") + ); + */ + /* + end + +rule "DOC.8.2: Performing Laboratory (Country & Name) from dict" + when + $facilityCountry: RedactionEntity(type=="testing_facility_country") + $facilityName: RedactionEntity(type=="testing_facility_name") + $section: Section( + ( + entities contains $facilityCountry + || entities contains $facilityName + ) + && ( + containsString("Testing Facility:") + || ( + containsString("Testing") + && containsString("Facility:") + ) + ) + ) + then + $section.getEntitiesOfType("testing_facility_country").forEach(entity -> { + entity.apply("DOC.8.2", "Testing facility country dictionary entry found."); + }); + $section.getEntitiesOfType("testing_facility_name").forEach(entity -> { + entity.apply("DOC.8.2", "Testing facility dictionary entry found."); + }); + end + + +rule "DOC.8.3: Performing Laboratory (Country) from dict" + when + $section: Section( + (hasEntitiesOfType("testing_facility_country") || hasEntitiesOfType("testing_facility_name")) + && !(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:"))) + ) + then + $section.getEntitiesOfType(List.of("testing_facility_country", "testing_facility_name")).forEach(entity -> { + entity.removeFromGraph(); + retract(entity); + }); + end + +rule "DOC.8.4: Test Facility Country with Ner Entities" + when + $countries: RedactionEntity(type=="ner_country") + $headline: Headline( + containsStringIgnoreCase("test facility") + ) + $section: Section( + getHeadline() == $headline + ) + $paragraph: Paragraph( + containsStringIgnoreCase("test facility") + && entities contains $countries + ) from $section.streamAllSubNodesOfType(NodeType.PARAGRAPH).toList() + then + $paragraph + .getEntitiesOfType("ner_country") + .forEach( + entity -> entityCreationService + .byBoundary( + entity.getBoundary(), + "testing_facility_country", + EntityType.ENTITY, + $paragraph + ) + .ifPresent( + country -> country.apply("DOC.8.4","Testing Facility Country found by NER") + ) + ); + end + +rule "DOC.9.0: Sponsor Name" + when + $headline: Headline( + containsStringIgnoreCase("sponsor") + ) + $section: Section( + getHeadline() == $headline + ) + $paragraph: Paragraph( + containsStringIgnoreCase("sponsor") + ) from $section.streamAllSubNodesOfType(NodeType.PARAGRAPH).toList() + then + entityCreationService + .semanticNodeAfterString( + "Sponsor", + "sponsor_name", + EntityType.ENTITY, + $paragraph + ) + .ifPresent( + entity -> entity.apply("DOC.9.0", "Sponsor found") + ); + end + + +rule "DOC.10.0: Active Ingredient Name" + when + $paragraph: Paragraph( + getHeadline().containsStringIgnoreCase("description") + && containsStringIgnoreCase("Content") + ) + then + entityCreationService + .byRegexIgnoreCase( + "\\b(\\w{5,20})\\b[\\s\\d.]{3,6}%", + "active_ingredient_name", + EntityType.ENTITY, + 1, + $paragraph + ) + .forEach( + entity -> entity.apply("DOC.10.0","Content names found.") + ); + end + +rule "DOC.11.0: Product Names" + when + $paragraph: Paragraph( + getHeadline().containsStringIgnoreCase("description") + && containsStringIgnoreCase("Content") + ) + then + entityCreationService + .byRegexIgnoreCase( + "\\b(\\w{5,20})\\b[\\s\\d.]{3,6}%", + "product_name", + EntityType.ENTITY, + 1, + $paragraph + ) + .forEach( + entity -> entity.apply("DOC.11.0","Product names found.") + ); + end + +rule "DOC.12.0: Section" + when + then + end + +rule "DOC.13.0: Guidelines" + when + $section: Section( + ( + containsString("according to") + || containsString("method") + ) + && ( + containsString("OECD") + || containsString("EPA") + || containsString("OPPTS") + || containsString("EC") + ) + ) + then + entityCreationService.byRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", "guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.13.0", "OECD Guideline found", "n-a") + ); + entityCreationService.byRegex("OECD[\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\d{3,4}(?>.{1,100}\\d{4}\\))?|\\[.{1,20}.Skin.{1,20}\\]|[\\d\\s,\\(\\)]{7,10}|[\\w\\.\\s]{1,15}[\\d]{3}\\s\\(\\d{4}\\)|.{0,20}[N|n]umber\\s\\d{3}.{0,1}|Test Guideline \\d{3})", "guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.13.0", "OECD Guideline found", "n-a") + ); + entityCreationService.byRegex("OECD [G|g]uideline 4\\d{2}", "guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.13.0", "OECD Guideline found", "n-a") + ); + + entityCreationService.byRegex("EC (Directive )?(No\\.? )?\\d{3,4}\\/\\d{3,4}((,? B(\\.| )\\d{1,2}\\.?)? \\(\\d{4}\\))?", "guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.13.0", "EC Guideline found", "n-a") + ); + entityCreationService.byRegex("EC method B(\\.| )\\d{1,2} ?\\w{0,5} \\([\\d\\/EC]{4,10}\\)?", "guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.13.0", "EC Guideline found", "n-a") + ); + entityCreationService.byRegex("Commission Regulation \\(EC\\) No \\d{3}\\/\\d{4}", "guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.13.0", "EC Guideline found", "n-a") + ); + + entityCreationService.byRegex("OPPTS (Guideline Number )?\\d{3}\\.\\d{4}( \\(\\d{4}\\))?", "guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.13.0", "EPA Guideline found", "n-a") + ); + entityCreationService.byRegex("EPA (OPPTS )?\\d{3}[. ]\\d{4}( \\(\\d{4}\\))?", "guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.13.0", "EPA Guideline found", "n-a") + ); + end +*/ +rule "DOC.14.0: GLP Study" + when + $hlKeywords: String() from List.of( + "Good Laboratory Practice", + "GLP" + ) + $headline: Headline( + containsString($hlKeywords) + ) + $compliance: String() from List.of( + "conducted in compliance with", + "conducted in accordance with", + "conducted and reported in compliance with", + "carried out in accordance with", + "meets the requirements", + "performed in compliance with" + ) + $section: Section( + containsStringIgnoreCase($compliance) + && getHeadline() == $headline + ) + then + entityCreationService.bySemanticNode($headline, "glp_study", EntityType.ENTITY).ifPresent(entity -> { + entity.apply("DOC.14.0", "GLP Study found", "n-a"); + insert(FileAttribute.builder().label("GLP").value("TRUE").build()); + }); + end + +rule "DOC.14.1: GLP Study" + when + $compliance: String() from List.of( + "conducted in compliance with", + "conducted in accordance with", + "conducted and reported in compliance with", + "carried out in accordance with", + "meets the requirements", + "performed in compliance with", + "conduct" + ) + $principle: String() from List.of( + "40 CFR Part 160", + "40CFR160", + "Good Laboratory Practice", + "FIFRA Good Laboratory Practice Standards" + ) + $paragraph: Paragraph( + containsStringIgnoreCase($compliance) + && containsStringIgnoreCase($principle) + ) + then + entityCreationService.betweenStringsIncludeStartAndEndIgnoreCase( + $compliance, + $principle, + "glp_study", + EntityType.ENTITY, + $paragraph + ) + .forEach(entity -> { + entity.apply("DOC.14.1", "GLP Study found", "n-a"); + insert(FileAttribute.builder().label("GLP").value("TRUE").build()); + } + ); + end + +//------------------------------------ Manual redaction rules ------------------------------------ + +// Rule unit: MAN.0 +rule "MAN.0.0: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId) + $entityToBeResized: RedactionEntity(matchesAnnotationId($id)) + then + manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($entityToBeResized); + end + + +// Rule unit: MAN.1 +rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" + salience 128 + when + IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) + then + $entityToBeRemoved.removeFromGraph(); + retract($entityToBeRemoved); + end + +rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" + salience 128 + when + IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $imageEntityToBeRemoved: Image($id == id) + then + $imageEntityToBeRemoved.setIgnored(true); + end + + +// Rule unit: MAN.2 +rule "MAN.2.0: Apply force redaction" + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $entityToForce: RedactionEntity(matchesAnnotationId($id)) + then + $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.setRemoved(false); + $entityToForce.setIgnored(false); + $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + update($entityToForce); + $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); + end + + +// Rule unit: MAN.3 +rule "MAN.3.0: Apply image recategorization" + salience 128 + when + ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) + $image: Image($id == id) + then + $image.setImageType(ImageType.fromString($imageType)); + end + + + +//------------------------------------ Entity merging rules ------------------------------------ + +// Rule unit: X.0 +rule "X.0.0: remove Entity contained by Entity of same type" + salience 65 + when + $larger: RedactionEntity($type: type, $entityType: entityType, isActive()) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) + then + $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); + retract($contained); + end + + +// Rule unit: X.7 +rule "X.7.0: remove all images" + salience 512 + when + $image: Image(imageType != ImageType.OCR) + then + $image.remove("X.7.0", "remove all images"); + retract($image); + end + + +//------------------------------------ File attributes rules ------------------------------------ + +// Rule unit: FA.1 +rule "FA.1.0: remove duplicate FileAttributes" + + salience 64 + when + $fileAttribute: FileAttribute($label: label, $value: value) + $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value) + then + retract($duplicate); + end + + +// Rule unit: LDS.0 +rule "LDS.0.0: run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) + .forEach(entity -> { + Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); + entity.addMatchedRules(matchedRules); + }); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index 0de1d713..7b3fdeb2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -1,8 +1,8 @@ package drools import static java.lang.String.format; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.anyMatch; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.exactMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; import java.util.List; import java.util.LinkedList; @@ -12,41 +12,47 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; -import com.iqser.red.service.redaction.v1.server.document.graph.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.*; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; -import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; global Document document global EntityCreationService entityCreationService @@ -1035,7 +1041,7 @@ rule "ETC.2.1: Redact signatures (vertebrate study)" FileAttribute(label == "Vertebrate Study", value == "Yes") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.apply("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -1095,7 +1101,7 @@ rule "ETC.7.0: Guidelines FileAttributes" when $section: Section(!hasTables(), (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS"))) then - RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream() + RedactionSearchUtility.findTextRangesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream() .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) .forEach(fileAttribute -> insert(fileAttribute)); @@ -1241,20 +1247,31 @@ rule "MAN.2.1: Apply force redaction to images" rule "MAN.3.0: Apply entity recategorization" salience 128 when - $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) then - if (!$recategorization.getType().equals($entityToBeRecategorized.getType())) { - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); - } + $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); + manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + retract($recategorization); + // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. + retract($entityToBeRecategorized); + end + + +rule "MAN.3.1: Apply entity recategorization of same type" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + then + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); end -rule "MAN.3.1: Apply image recategorization" + +rule "MAN.3.2: Apply image recategorization" salience 128 when $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) @@ -1267,6 +1284,7 @@ rule "MAN.3.1: Apply image recategorization" retract($recategorization); end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 16a5dab3..d65f54e4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1,3 +1,9 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + import java.util.List; import java.util.LinkedList; import java.util.Set; @@ -6,41 +12,47 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; -import com.iqser.red.service.redaction.v1.server.document.graph.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.*; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; -import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; global Document document global EntityCreationService entityCreationService @@ -57,7 +69,7 @@ query "getFileAttributes" -rule "H.0.0 retract table of contents page" +rule "H.0.0: retract table of contents page" when $page: Page(getMainBodyTextBlock().getSearchText().contains("........") || (getMainBodyTextBlock().getSearchText().contains("APPENDICES") && getMainBodyTextBlock().getSearchText().contains("TABLES"))) $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE) @@ -66,7 +78,7 @@ rule "H.0.0 retract table of contents page" end -rule "H.0.0: Ignore Table of Contents" +rule "H.0.1: Ignore Table of Contents" salience 10 when $tocHeadline: Headline(containsString("CONTENTS")) @@ -86,12 +98,13 @@ rule "H.0.0: Ignore Table of Contents" rule "H.0.0: Show headlines" when $headline: Headline() + $headline: Headline() then entityCreationService.bySemanticNode($headline, "headline", EntityType.RECOMMENDATION); end */ -rule "H.0.0: Study Type File Attribute" +rule "H.0.2: Study Type File Attribute" when not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","408","414","425","429","436","438","439","471","487")) $section: Section( @@ -99,10 +112,10 @@ rule "H.0.0: Study Type File Attribute" ,(containsString("OECD") || containsString("EPA") || containsString("OPPTS")) ) then - Stream.of(RedactionSearchUtility.findBoundariesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1, $section.getTextBlock()), - RedactionSearchUtility.findBoundariesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()), - RedactionSearchUtility.findBoundariesByRegexIgnoreCase("(?<=OECD) Guideline (4\\d{2})", 1, $section.getTextBlock()), - RedactionSearchUtility.findBoundariesByRegexIgnoreCase("(?<=OECD) Guideline, Method No. (\\d{3})", 1, $section.getTextBlock()) + Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1, $section.getTextBlock()), + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()), + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD) Guideline (4\\d{2})", 1, $section.getTextBlock()), + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD) Guideline, Method No. (\\d{3})", 1, $section.getTextBlock()) ).flatMap(Collection::stream).findFirst() .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) @@ -252,22 +265,22 @@ rule "DOC.3.2: Experimental Completion Date" then entityCreationService.byRegex("STUDY COMPLETED ON (.{5,20}\\d{4})", "experimental_end_date", EntityType.ENTITY, 1, $section).forEach(entity -> { - entity.apply("DOC.3.1", "Experimental end date found", "n-a"); + entity.apply("DOC.3.2", "Experimental end date found", "n-a"); }); entityCreationService.byRegex("STUDY COMPLETION DATE (.{5,20}\\d{4})", "experimental_end_date", EntityType.ENTITY, 1, $section).forEach(entity -> { - entity.apply("DOC.3.1", "Experimental end date found", "n-a"); + entity.apply("DOC.3.2", "Experimental end date found", "n-a"); }); entityCreationService.byRegex("Report completion date (.{5,20}\\d{4})", "experimental_end_date", EntityType.ENTITY, 1, $section).forEach(entity -> { - entity.apply("DOC.3.1", "Experimental end date found", "n-a"); + entity.apply("DOC.3.2", "Experimental end date found", "n-a"); }); entityCreationService.byRegex("Date of Report (.{5,20}\\d{4})", "experimental_end_date", EntityType.ENTITY, 1, $section).forEach(entity -> { - entity.apply("DOC.3.1", "Experimental end date found", "n-a"); + entity.apply("DOC.3.2", "Experimental end date found", "n-a"); }); entityCreationService.byRegex("AMENDMENT COMPLETION DATE (.{5,20}\\d{4})", "experimental_end_date", EntityType.ENTITY, 1, $section).forEach(entity -> { - entity.apply("DOC.3.1", "Experimental end date found", "n-a"); + entity.apply("DOC.3.2", "Experimental end date found", "n-a"); }); entityCreationService.byRegex("AMENDMENT COMPLETED ON (.{5,20}\\d{4})", "experimental_end_date", EntityType.ENTITY, 1, $section).forEach(entity -> { - entity.apply("DOC.3.1", "Experimental end date found", "n-a"); + entity.apply("DOC.3.2", "Experimental end date found", "n-a"); }); end @@ -365,20 +378,20 @@ rule "DOC.6.2: Authors" $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains "AUTHOR(S)" || contains "AUTHORS" || contains "Author"), getMainBodyTextBlock().getSearchText() (contains "STUDY COMPLETED ON" || contains "STUDY COMPLETION DATE" || contains "DATE OF INTERIM REPORT" || contains "Report completion date" || contains "Date of Report" || contains "AMENDMENT COMPLETION DATE")) then - List startBoundaries = new LinkedList<>(); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("AUTHOR(S)", $page.getMainBodyTextBlock())); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("AUTHORS", $page.getMainBodyTextBlock())); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("Author", $page.getMainBodyTextBlock())); + List startBoundaries = new LinkedList<>(); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("AUTHOR(S)", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("AUTHORS", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author", $page.getMainBodyTextBlock())); - List stopBoundaries = new LinkedList<>(); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("STUDY COMPLETED ON", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("STUDY COMPLETION DATE", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("DATE OF INTERIM REPORT", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("Report completion date", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("Date of Report", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("AMENDMENT COMPLETION DATE", $page.getMainBodyTextBlock())); + List stopBoundaries = new LinkedList<>(); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("STUDY COMPLETED ON", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("STUDY COMPLETION DATE", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("DATE OF INTERIM REPORT", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Report completion date", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Date of Report", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("AMENDMENT COMPLETION DATE", $page.getMainBodyTextBlock())); - entityCreationService.betweenBoundaries(startBoundaries, stopBoundaries, "author", EntityType.ENTITY, document).forEach(entity -> { + entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "author", EntityType.ENTITY, document).forEach(entity -> { entity.apply("DOC.6.2", "Author found", "n-a"); }); end @@ -390,16 +403,16 @@ rule "DOC.6.6: laboratory_project_identification" when $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains "LABORATORY PROJECT IDENTIFICATION" || contains "TEST FACILITY PROJECT IDENTIFICATION" || contains "Laboratory Project Identification")) then - List startBoundaries = new LinkedList<>(); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("LABORATORY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("TEST FACILITY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); + List startBoundaries = new LinkedList<>(); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("LABORATORY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST FACILITY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); - List stopBoundaries = new LinkedList<>(); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("SPONSOR", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("VOLUME", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("This", $page.getMainBodyTextBlock())); + List stopBoundaries = new LinkedList<>(); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("SPONSOR", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("VOLUME", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("This", $page.getMainBodyTextBlock())); - entityCreationService.betweenBoundaries(startBoundaries, stopBoundaries, "laboratory_project_identification", EntityType.ENTITY, document).forEach(entity -> { + entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "laboratory_project_identification", EntityType.ENTITY, document).forEach(entity -> { entity.apply("DOC.6.6", "Laboratory Project Identification", "n-a"); }); end @@ -411,50 +424,50 @@ rule "DOC.7.2: study title by document structure" $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains "STUDY TITLE" || contains "Study Title" || contains "STUDYTITLE" || contains "Report (Final)")) then - List startBoundaries = new LinkedList<>(); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("STUDY TITLE", $page.getMainBodyTextBlock())); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("STUDYTITLE", $page.getMainBodyTextBlock())); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("Report (Final)", $page.getMainBodyTextBlock())); + List startBoundaries = new LinkedList<>(); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("STUDY TITLE", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("STUDYTITLE", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Report (Final)", $page.getMainBodyTextBlock())); - List stopBoundaries = new LinkedList<>(); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("TEST GUIDELINES", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("TEST GUIDELINE(S)", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("Guidelines", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("DATA REQUIREMENT", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("AUTHOR(S)", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("AUTHOR", $page.getMainBodyTextBlock())); + List stopBoundaries = new LinkedList<>(); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST GUIDELINES", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST GUIDELINE(S)", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Guidelines", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("DATA REQUIREMENT", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("AUTHOR(S)", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("AUTHOR", $page.getMainBodyTextBlock())); - entityCreationService.betweenBoundaries(startBoundaries, stopBoundaries, "title", EntityType.ENTITY, document).forEach(entity -> { + entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "title", EntityType.ENTITY, document).forEach(entity -> { entity.apply("DOC.7.2", "Study title found", "n-a"); }); end -rule "DOC.8.2: Performing Laboratory" +rule "DOC.8.1: Performing Laboratory" when $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains "PERFORMING LABORATORY" || contains "TEST FACILITIES" || contains "TEST FACILITY" || contains "Test Facility"), getMainBodyTextBlock().getSearchText() (contains "LABORATORY PROJECT IDENTIFICATION" || contains "TEST FACILITY PROJECT IDENTIFICATION" || contains "Sponsor")) then - List startBoundaries = new LinkedList<>(); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("PERFORMING LABORATORY", $page.getMainBodyTextBlock())); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("TEST FACILITIES", $page.getMainBodyTextBlock())); - startBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("TEST FACILITY", $page.getMainBodyTextBlock())); + List startBoundaries = new LinkedList<>(); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("PERFORMING LABORATORY", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST FACILITIES", $page.getMainBodyTextBlock())); + startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST FACILITY", $page.getMainBodyTextBlock())); - List stopBoundaries = new LinkedList<>(); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("LABORATORY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("TEST FACILITY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("Sponsor", $page.getMainBodyTextBlock())); - stopBoundaries.addAll(RedactionSearchUtility.findBoundariesByStringIgnoreCase("PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); + List stopBoundaries = new LinkedList<>(); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("LABORATORY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("TEST FACILITY PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Sponsor", $page.getMainBodyTextBlock())); + stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("PROJECT IDENTIFICATION", $page.getMainBodyTextBlock())); - entityCreationService.betweenBoundaries(startBoundaries, stopBoundaries, "laboratory_name", EntityType.ENTITY, document).forEach(entity -> { - entity.apply("DOC.8.2", "Performing Laboratory found", "n-a"); + entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "laboratory_name", EntityType.ENTITY, document).forEach(entity -> { + entity.apply("DOC.8.1", "Performing Laboratory found", "n-a"); }); end -rule "DOC.98.2: Summary Methods" +rule "DOC.8.2: Summary Methods" when $headline: Headline(containsString("1.1. METHODS")) then @@ -469,7 +482,7 @@ rule "DOC.98.2: Summary Methods" .forEach(entity -> entity.apply("DOC.8.2", "Summary Methods found", "n-a")); end -rule "DOC.98.3: Summary Observations Laboratory" +rule "DOC.8.3: Summary Observations Laboratory" when $headline: Headline(containsString("1.2. OBSERVATIONS")) then @@ -485,7 +498,7 @@ rule "DOC.98.3: Summary Observations Laboratory" end -rule "DOC.98.5: Summary Results" +rule "DOC.8.5: Summary Results" when Headline((containsStringIgnoreCase("1.3. RESULTS") || containsStringIgnoreCase("1.2. RESULTS")), $sectionIdentifier: getSectionIdentifier()) $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier)) @@ -503,7 +516,7 @@ rule "DOC.98.5: Summary Results" }); end -rule "DOC.98.6: Summary Results 2" +rule "DOC.8.6: Summary Results 2" when $headline: Headline(containsString("1.2. RESULTS")) then @@ -515,12 +528,12 @@ rule "DOC.98.6: Summary Results 2" .filter(e -> !e.getValue().startsWith("BASF")) .filter(e -> !e.getValue().startsWith("The Chemical Company")) .filter(e -> !e.getValue().startsWith("We create chemistry")) - .forEach(entity -> entity.apply("DOC.8.5", "Summary Results", "n-a")); + .forEach(entity -> entity.apply("DOC.8.6", "Summary Results", "n-a")); end -rule "DOC.98.4: Summary Conclusion" +rule "DOC.8.4: Summary Conclusion" when $headline: Headline(containsString("1.4. CONCLUSION") || containsString("1.3. CONCLUSION")) then @@ -532,7 +545,7 @@ rule "DOC.98.4: Summary Conclusion" .filter(e -> !e.getValue().startsWith("BASF")) .filter(e -> !e.getValue().startsWith("The Chemical Company")) .filter(e -> !e.getValue().startsWith("We create chemistry")) - .forEach(entity -> entity.apply("DOC.8.3", "Summary Conculsion found", "n-a")); + .forEach(entity -> entity.apply("DOC.8.4", "Summary Conculsion found", "n-a")); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl new file mode 100644 index 00000000..54c02fd1 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl @@ -0,0 +1,235 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; + +global ComponentCreationService componentCreationService + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +query "getComponents" + $component: Component() + end + +//------------------------------------ Guideline mapping object ------------------------------------ + +declare GuidelineMapping + number: String + year: String + guideline: String + end + +//------------------------------------ Default Components rules ------------------------------------ + +rule "StudyTitle.0.0: Study Title" + when + $titleCandidates: List() from collect (Entity(type == "study_title")) + then + componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!"); + end + + +rule "PerformingLaboratory.1.0: Performing Laboratory" + when + $laboratoryName: Entity(type == "laboratory_name") + $laboratoryCountry: Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80) + not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset)) + then + componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!"); + end + +rule "PerformingLaboratory.2.0: Performing Laboratory" + when + $laboratoryName: Entity(type == "laboratory_name") + not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80) + then + componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!"); + end + +rule "PerformingLaboratory.0.2: Performing Laboratory" + salience -1 + when + not Component(category == "Performing_Laboratory") + then + componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback"); + end + + +rule "ReportNumber.0.0: Report number" + when + $reportNumberCandidates: List() from collect (Entity(type == "report_number")) + then + componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "No report number found!"); + end + + +rule "GLPStudy.0.0: GLP Study" + when + $glpStudy: Entity(type == "glp_study") + then + componentCreationService.create("GLPStudy.0.0", "GLP_study", "Yes", "Yes if present, No if not", $glpStudy); + end + +rule "GLPStudy.1.0: GLP Study" + when + not Entity(type == "glp_study") + then + componentCreationService.create("GLPStudy.1.0", "GLP_study", "No", "Yes if present, No if not"); + end + +rule "GLPStudy.2.0: GLP Study" + when + $first: Component(category == "GLP_study") + $second: Component(category == "GLP_study", value == $first.value, this != $first) + then + $first.addReferences($second.getReferences()); + retract($second); + end + + +rule "TestGuideline.0.0: no test guideline year found, only oecd number" + when + $guidelineNumber: Entity(type == "oecd_guideline_number") + not Entity(type == "oecd_guideline_year") + then + componentCreationService.create("DefaultComponents.4.0", "Test_Guidelines_1", $guidelineNumber.getValue(), "Only OECD Number found!"); + end + +rule "TestGuideline.1.0: create mappings" +salience 1 + when + Entity(type == "oecd_guideline_number") + Entity(type == "oecd_guideline_year") + then + insert(new GuidelineMapping("425", "2008", "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (03/10/2008)")); + insert(new GuidelineMapping("425", "2001", "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (17/12/2001)")); + insert(new GuidelineMapping("402", "2017", "Nº 402: Acute Dermal Toxicity (09/10/2017)")); + insert(new GuidelineMapping("402", "1987", "Nº 402: Acute Dermal Toxicity (24/02/1987)")); + insert(new GuidelineMapping("403", "2009", "Nº 403: Acute Inhalation Toxicity (08/09/2009)")); + insert(new GuidelineMapping("403", "1981", "Nº 403: Acute Inhalation Toxicity (12/05/1981)")); + insert(new GuidelineMapping("433", "2018", "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (27/06/2018)")); + insert(new GuidelineMapping("433", "2017", "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (09/10/2017)")); + insert(new GuidelineMapping("436", "2009", "Nº 436: Acute Inhalation Toxicity – Acute Toxic Class Method (08/09/2009)")); + insert(new GuidelineMapping("404", "1981", "Nº 404: Acute Dermal Irritation/Corrosion (12/05/1981)")); + insert(new GuidelineMapping("404", "1992", "Nº 404: Acute Dermal Irritation/Corrosion (17/07/1992)")); + insert(new GuidelineMapping("404", "2002", "Nº 404: Acute Dermal Irritation/Corrosion (24/04/2002)")); + insert(new GuidelineMapping("404", "2015", "Nº 404: Acute Dermal Irritation/Corrosion (28/07/2015)")); + insert(new GuidelineMapping("405", "2017", "Nº 405: Acute Eye Irritation/Corrosion (09/10/2017)")); + insert(new GuidelineMapping("405", "2012", "Nº 405: Acute Eye Irritation/Corrosion (02/10/2012)")); + insert(new GuidelineMapping("405", "2002", "Nº 405: Acute Eye Irritation/Corrosion (24/04/2002)")); + insert(new GuidelineMapping("405", "1987", "Nº 405: Acute Eye Irritation/Corrosion (24/02/1987)")); + insert(new GuidelineMapping("429", "2002", "Nº 429: Skin Sensitisation: Local Lymph Node Assay (24/04/2002)")); + insert(new GuidelineMapping("429", "2010", "Nº 429: Skin Sensitisation (23/07/2010)")); + insert(new GuidelineMapping("442A", "2018", "Nº 442A: Skin Sensitization (23/07/2018)")); + insert(new GuidelineMapping("442B", "2018", "Nº 442B: Skin Sensitization (27/06/2018)")); + insert(new GuidelineMapping("471", "1997", "Nº 471: Bacterial Reverse Mutation Test (21/07/1997)")); + insert(new GuidelineMapping("471", "2020", "Nº 471: Bacterial Reverse Mutation Test (26/06/2020)")); + insert(new GuidelineMapping("406", "1992", "Nº 406: Skin Sensitisation (1992)")); + insert(new GuidelineMapping("428", "2004", "Nº 428: Split-Thickness Skin test (2004)")); + insert(new GuidelineMapping("438", "2018", "Nº 438: Eye Irritation (26/06/2018)")); + insert(new GuidelineMapping("439", "2019", "Nº 439: Skin Irritation (2019)")); + insert(new GuidelineMapping("474", "2016", "Nº 474: Micronucleus Bone Marrow Cells Rat (2016)")); + insert(new GuidelineMapping("487", "2016", "Nº 487: Micronucleus Human Lymphocytes (2016)")); + end + +rule "TestGuideline.1.1: match test guidelines with mappings" + when + GuidelineMapping($year: year, $number: number, $guideline: guideline) + $guidelineNumber: Entity(type == "oecd_guideline_number", value == $number) + $guidelineYear: Entity(type == "oecd_guideline_year", value == $year) + then + componentCreationService.create( + "DefaultComponents.4.1", + "Test_Guidelines_1", + $guideline, + "OECD Number and guideline year mapped!" + ); + end + + +rule "DefaultComponents.5.0: Test Guideline 2" + when + $epaGuideLines: List() from collect (Entity(type == "epa_guideline")) + $ecGuideLines: List() from collect (Entity(type == "ec_guideline")) + then + componentCreationService.joining("DefaultComponents.5.0", + "Test_Guideline_2", + Stream.of( + $epaGuideLines.stream(), + $ecGuideLines.stream()) + .flatMap(a -> a) + .toList() + ); + end + + +rule "DefaultComponents.6.0: Experimental Starting Date" + when + $startDates: List(!isEmpty()) from collect (Entity(type == "experimental_start_date")) + then + componentCreationService.convertDates("DefaultComponents.6.0", "Experimental_Starting_Date", $startDates); + end + + +rule "DefaultComponents.7.0: Experimental Completion Date" + when + $endDates: List(!isEmpty()) from collect (Entity(type == "experimental_end_date")) + then + componentCreationService.convertDates("DefaultComponents.7.0", "Experimental_Completion_Date", $endDates); + end + + +rule "DefaultComponents.8.0: Certificate of analysis batch identification" + when + $batchNumbers: List(!isEmpty()) from collect (Entity(type == "batch_number")) + then + componentCreationService.joiningUnique("DefaultComponents.8.0", "Batch_Number", $batchNumbers); + end + + +rule "DefaultComponents.999.0: Create components for all unmapped entities." + salience -999 + when + $allEntities: List(!isEmpty()) from collect (Entity()) + then + componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities); + end + + + +//------------------------------------ Component merging rules ------------------------------------ + +rule "X.0.0: merge duplicate component references" + when + $first: Component() + $duplicate: Component(this != $first, category == category, value == value) + then + $first.getReferences().addAll($duplicate.getReferences()); + retract($duplicate); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 882f0579..94acedec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -1,8 +1,8 @@ package drools import static java.lang.String.format; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.anyMatch; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.exactMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; import java.util.List; import java.util.LinkedList; @@ -12,41 +12,47 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; -import com.iqser.red.service.redaction.v1.server.document.graph.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.*; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; -import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; global Document document global EntityCreationService entityCreationService diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 0e6dc251..7227d3b1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1,8 +1,8 @@ package drools import static java.lang.String.format; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.anyMatch; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.exactMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; import java.util.List; import java.util.LinkedList; @@ -12,41 +12,47 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; -import com.iqser.red.service.redaction.v1.server.document.graph.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.*; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; -import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; global Document document global EntityCreationService entityCreationService @@ -760,7 +766,7 @@ rule "ETC.2.1: Redact signatures (vertebrate study)" FileAttribute(label == "Vertebrate Study", value == "Yes") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.apply("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -820,7 +826,7 @@ rule "ETC.7.0: Guidelines FileAttributes" when $section: Section(!hasTables(), (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS"))) then - RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream() + RedactionSearchUtility.findTextRangesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream() .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) .forEach(fileAttribute -> insert(fileAttribute)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index c32bcb56..198f38cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -1,8 +1,8 @@ package drools import static java.lang.String.format; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.anyMatch; -import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.exactMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; import java.util.List; import java.util.LinkedList; @@ -12,46 +12,53 @@ import java.util.Collection; import java.util.stream.Stream; import java.util.Optional; -import com.iqser.red.service.redaction.v1.server.document.graph.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.*; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; -import com.iqser.red.service.redaction.v1.server.document.services.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextRange; -import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType -import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType -import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary + // --------------------------------------- queries ------------------------------------------------------------------- query "getFileAttributes" @@ -60,18 +67,19 @@ query "getFileAttributes" // --------------------------------------- NER Entities rules ------------------------------------------------------------------- -rule "add NER Entities of type CBI_author or CBI_address" +// Rule unit: AI.0 +rule "AI.0.0: add all NER Entities of type CBI_author" salience 999 when - $nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address")) + nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then - entityCreationService.byTextRange(new TextRange($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document) - .ifPresent(redactionEntity -> insert(redactionEntity)); + nerEntities.streamEntitiesOfType("CBI_author") + .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end // --------------------------------------- CBI rules ------------------------------------------------------------------- -rule "Always redact CBI_author" +rule "CBI.0.0: Always redact CBI_author" when $cbiAuthor: TextEntity(type == "CBI_author", entityType == EntityType.ENTITY) @@ -81,7 +89,7 @@ rule "Always redact CBI_author" // --------------------------------------- PII rules ------------------------------------------------------------------- -rule "Always redact PII" +rule "PII.0.0: Always redact PII" when $cbiAuthor: TextEntity(type == "PII", entityType == EntityType.ENTITY) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index e8f074d2..1a57e91b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -25,7 +25,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; @@ -45,7 +45,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType -import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType global Document document global EntityCreationService entityCreationService