diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RuleFileType.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RuleFileType.java new file mode 100644 index 00000000..d47785bf --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RuleFileType.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.redaction.v1.model; + +public enum RuleFileType { + + ENTITY, + COMPONENT +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rules.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rules.java new file mode 100644 index 00000000..11fd5ee9 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rules.java @@ -0,0 +1,15 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class Rules { + + RuleFileType ruleFileType; + String rulesString; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index 177240a1..c2cc1409 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -5,10 +5,11 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.Rules; public interface RedactionResource { @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE) - DroolsSyntaxValidation testRules(@RequestBody String rules); + DroolsSyntaxValidation testRules(@RequestBody Rules rules); } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 17f32180..d99bb69b 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.25.0" val jacksonVersion = "2.15.2" val droolsVersion = "8.43.0.Final" val pdfBoxVersion = "3.0.0-alpha2" -val persistenceServiceVersion = "2.160.0" +val persistenceServiceVersion = "DM-285-2" configurations { all { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 6d8f83c8..e268eca8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -4,6 +4,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.Rules; import com.iqser.red.service.redaction.v1.resources.RedactionResource; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService; import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException; @@ -20,7 +21,7 @@ public class RedactionController implements RedactionResource { @Override - public DroolsSyntaxValidation testRules(@RequestBody String rules) { + public DroolsSyntaxValidation testRules(@RequestBody Rules rules) { try { return droolsSyntaxValidationService.testRules(rules); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java index e9889d4f..617729e5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java @@ -22,7 +22,7 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ManualEntity implements Entity { - // must be mapped into a TextEntity as is for comments to work correctly + // The id must be mapped into a TextEntity as is for comments to work correctly String id; String value; List entityPosition; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java new file mode 100644 index 00000000..98fce44f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Component.java @@ -0,0 +1,24 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class Component { + + String value; + String originalValue; + String transformation; + + List references; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index 455dc26b..cace9fb2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.PriorityQueue; import java.util.Set; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; @@ -50,7 +50,6 @@ public class TextEntity implements Entity { @Builder.Default Set engines = new HashSet<>(); - // inferred on graph insertion String value; String textBefore; @@ -201,4 +200,6 @@ public class TextEntity implements Entity { return sb.toString(); } + + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index cd16f262..0b4a2b0e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -14,6 +14,10 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; @@ -21,21 +25,24 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; -import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; -import com.iqser.red.service.redaction.v1.server.service.graph.DocumentGraphMapper; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; -import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; +import com.iqser.red.service.redaction.v1.server.model.KieWrapper; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Component; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; +import com.iqser.red.service.redaction.v1.server.service.graph.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.graph.ManualRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.graph.NerEntitiesAdapter; -import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import io.micrometer.core.annotation.Timed; @@ -55,9 +62,12 @@ public class AnalyzeService { DictionaryService dictionaryService; EntityDroolsExecutionService entityDroolsExecutionService; + ComponentDroolsExecutionService componentDroolsExecutionService; KieContainerCreationService kieContainerCreationService; DictionarySearchService dictionarySearchService; + RedactionLogCreatorService redactionLogCreatorService; EntityLogCreatorService entityLogCreatorService; + ComponentLogCreatorService componentLogCreatorService; RedactionStorageService redactionStorageService; RedactionChangeLogService redactionChangeLogService; LegalBasisClient legalBasisClient; @@ -101,43 +111,63 @@ public class AnalyzeService { analyzeRequest.getFileAttributes(), analyzeRequest.getManualRedactions(), nerEntities); + + List components = componentDroolsExecutionService.executeRules(document.getEntities(), + addedFileAttributes, + notFoundManualRedactionEntries, + document.streamAllImages().toList()); + + ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components); + log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - List redactionLogEntries = entityLogCreatorService.createRedactionLog(document, - analyzeRequest.getDossierTemplateId(), - notFoundManualRedactionEntries, - getComments(analyzeRequest)); + RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, wrapper); + EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, wrapper); + + return finalizeAnalysis(analyzeRequest, + startTime, + entityLog, + redactionLog, + document.getNumberOfPages(), + dictionary.getVersion(), + false, + new HashSet<>(addedFileAttributes)); + } + + + private EntityLog createEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + Dictionary dictionary, + KieWrapper wrapper) { + + List entityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), + + EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), - redactionLogEntries, - toSimplifiedSectionText(legalBasis), + entityLogEntries, + toEntityLogLegalBasis(legalBasis), dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(), wrapper.rulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); - List importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), + List importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), - redactionLog.getRedactionLogEntry(), + entityLog.getEntityLogEntry(), true); - redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries); - return finalizeAnalysis(analyzeRequest, startTime, redactionLog, document.getNumberOfPages(), dictionary.getVersion(), false, new HashSet<>(addedFileAttributes)); + entityLog.setEntityLogEntry(importedRedactionFilteredEntries); + return entityLog; } - private static Map> getComments(AnalyzeRequest analyzeRequest) { + private List toEntityLogLegalBasis(List legalBasis) { - if (analyzeRequest.getManualRedactions() == null) { - return Collections.emptyMap(); - } - if (analyzeRequest.getManualRedactions().getComments() == null) { - return Collections.emptyMap(); - } - return analyzeRequest.getManualRedactions().getComments(); + return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); } @@ -147,13 +177,14 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); // not yet ready for reanalysis - if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) { + if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) { return analyze(analyzeRequest); } @@ -168,6 +199,7 @@ public class AnalyzeService { if (sectionsToReAnalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, startTime, + previousEntityLog, previousRedactionLog, document.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), @@ -198,7 +230,28 @@ public class AnalyzeService { nerEntities); log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - List newRedactionLogEntries = entityLogCreatorService.createRedactionLog(document, + RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); + EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds); + + return finalizeAnalysis(analyzeRequest, + startTime, + entityLog, + redactionLog, + document.getNumberOfPages(), + dictionaryIncrement.getDictionaryVersion(), + true, + new HashSet<>(allFileAttributes)); + } + + + @Deprecated(forRemoval = true) + private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + RedactionLog previousRedactionLog, + Set sectionsToReanalyseIds) { + + List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries, getComments(analyzeRequest)); @@ -212,19 +265,34 @@ public class AnalyzeService { previousRedactionLog.getRedactionLogEntry() .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)); previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries); + return previousRedactionLog; + } - return finalizeAnalysis(analyzeRequest, - startTime, - previousRedactionLog, - document.getNumberOfPages(), - dictionaryIncrement.getDictionaryVersion(), - true, - new HashSet<>(allFileAttributes)); + + private EntityLog updatePreviousEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + EntityLog previousEntityLog, + Set sectionsToReanalyseIds) { + + List newEntityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + + var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + newEntityLogEntries, + false); + + previousEntityLog.getEntityLogEntry() + .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)); + previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries); + return previousEntityLog; } private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, + EntityLog entityLog, RedactionLog redactionLog, int numberOfPages, DictionaryVersion dictionaryVersion, @@ -233,7 +301,6 @@ public class AnalyzeService { redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion()); redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion()); - excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages()); RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), @@ -316,7 +383,51 @@ public class AnalyzeService { } - public List toSimplifiedSectionText(List legalBasis) { + @Deprecated(forRemoval = true) + private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundManualRedactionEntries, + Dictionary dictionary, + KieWrapper wrapper) { + + List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, + analyzeRequest.getDossierTemplateId(), + notFoundManualRedactionEntries, + getComments(analyzeRequest)); + + List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); + RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), + analyzeRequest.getAnalysisNumber(), + redactionLogEntries, + toRedactionLogLegalBasis(legalBasis), + dictionary.getVersion().getDossierTemplateVersion(), + dictionary.getVersion().getDossierVersion(), + wrapper.rulesVersion(), + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + + List importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), + analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + redactionLog.getRedactionLogEntry(), + true); + redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries); + return redactionLog; + } + + + private static Map> getComments(AnalyzeRequest analyzeRequest) { + + if (analyzeRequest.getManualRedactions() == null) { + return Collections.emptyMap(); + } + if (analyzeRequest.getManualRedactions().getComments() == null) { + return Collections.emptyMap(); + } + return analyzeRequest.getManualRedactions().getComments(); + } + + + public List toRedactionLogLegalBasis(List legalBasis) { return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentCreationService.java deleted file mode 100644 index 7211fd2e..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentCreationService.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.service; - -public class ComponentCreationService { - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java new file mode 100644 index 00000000..b5b1087f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java @@ -0,0 +1,83 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Component; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; + +@Service +public class ComponentLogCreatorService { + + public ComponentLog buildComponentLog(int analysisNumber, List components) { + + List componentLogEntries = components.stream().map(this::buildComponentLogEntry).toList(); + return new ComponentLog(analysisNumber, componentLogEntries, -1, -1, -1); + } + + + private ComponentLogEntry buildComponentLogEntry(Component component) { + + return ComponentLogEntry.builder() + .value(component.getValue()) + .transformation(component.getTransformation()) + .originalValue(component.getOriginalValue()) + .componentEntityReferences(toComponentEntityReferences(component.getReferences())) + .build(); + } + + + private List toComponentEntityReferences(List references) { + + return references.stream().flatMap(this::toComponentEntityReference).toList(); + } + + + private Stream toComponentEntityReference(Entity entity) { + + if (entity instanceof TextEntity textEntity) { + textEntity.getPositionsOnPagePerPage() + .stream() + .map(positionOnPage -> ComponentEntityReference.builder() + .page(positionOnPage.getPage().getNumber()) + .type(textEntity.getType()) + .id(positionOnPage.getId()) + .ruleIdentifier(textEntity.getMatchedRule().getRuleIdentifier().toString()) + .reason(textEntity.buildReasonWithManualChangeDescriptions()) + .build()); + } else if (entity instanceof Image image) { + return Stream.of(ComponentEntityReference.builder() + .id(image.getId()) + .reason(image.buildReasonWithManualChangeDescriptions()) + .type(image.getImageType().toString()) + .ruleIdentifier(image.getMatchedRule().getRuleIdentifier().toString()) + .page(image.getFirstPage().getNumber()) + .build()); + } else if (entity instanceof ManualEntity manualEntity) { + return manualEntity.getEntityPosition() + .stream() + .collect(Collectors.groupingBy(RectangleWithPage::pageNumber)) + .keySet() + .stream() + .map(pageNumber -> ComponentEntityReference.builder() + .id(manualEntity.getId()) + .reason(manualEntity.buildReasonWithManualChangeDescriptions()) + .type(manualEntity.getType()) + .ruleIdentifier(manualEntity.getRuleIdentifier()) + .page(pageNumber) + .build()); + } + throw new UnsupportedOperationException(String.format("Entity with class %s is not implemented yet", entity.getClass())); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 442dde7d..26ed35ba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index f9c25cc4..388ccba8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -6,24 +6,23 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Entity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -37,19 +36,16 @@ public class EntityLogCreatorService { private final ManualChangeFactory manualChangeFactory; - public List createRedactionLog(Document document, - String dossierTemplateId, - List notFoundManualRedactionEntries, - Map> comments) { + public List createEntityLogEntries(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { - List entries = new ArrayList<>(); + List entries = new ArrayList<>(); document.getEntities() .stream() .filter(EntityLogCreatorService::isEntityOrRecommendationType) .filter(entity -> !entity.removed()) - .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments))); - document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments))); - notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments))); + .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId))); + document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); + notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId))); return entries; } @@ -60,10 +56,10 @@ public class EntityLogCreatorService { } - private List toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map> comments) { + private List toEntityLogEntries(TextEntity textEntity, String dossierTemplateId) { Set processedIds = new HashSet<>(); - List redactionLogEntities = new ArrayList<>(); + List redactionLogEntities = new ArrayList<>(); for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { @@ -73,11 +69,10 @@ public class EntityLogCreatorService { } processedIds.add(positionOnPage.getId()); - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId); + EntityLogEntry redactionLogEntry = createEntityLogEntry(textEntity, dossierTemplateId); redactionLogEntry.setId(positionOnPage.getId()); - redactionLogEntry.setComments(buildRedactionLogComments(comments, positionOnPage.getId())); - List rectanglesPerLine = positionOnPage.getRectanglePerLine() + List rectanglesPerLine = positionOnPage.getRectanglePerLine() .stream() .map(rectangle2D -> toRedactionLogRectangle(rectangle2D, positionOnPage.getPage().getNumber())) .toList(); @@ -90,53 +85,18 @@ public class EntityLogCreatorService { } - private List buildRedactionLogComments(Map> commentsPerId, String id) { - - if (!commentsPerId.containsKey(id)) { - return Collections.emptyList(); - } - List comments = commentsPerId.get(id); - if (comments == null || comments.isEmpty()) { - return Collections.emptyList(); - } - return toRedactionLogComments(comments); - } - - - private List toRedactionLogComments(List comments) { - - return comments.stream().map(this::toRedactionLogComment).toList(); - } - - - private RedactionLogComment toRedactionLogComment(Comment comment) { - - return new RedactionLogComment(comment.getId(), - comment.getUser(), - comment.getText(), - comment.getAnnotationId(), - comment.getFileId(), - comment.getDate(), - comment.getSoftDeletedTime()); - } - - - private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) { + private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) { Set referenceIds = new HashSet<>(); entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); boolean isHint = isHint(entity.getType(), dossierTemplateId); - return RedactionLogEntry.builder() + return EntityLogEntry.builder() .color(getColor(entity.getType(), dossierTemplateId, entity.applied())) .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .type(entity.getType()) - .redacted(entity.applied()) - .isHint(isHint) - .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION)) - .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) .sectionNumber(sectionNumber) .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) @@ -149,25 +109,25 @@ public class EntityLogCreatorService { .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) .reference(referenceIds) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) + .state(buildEntryState(entity)) + .entryType(buildEntryType(entity, isHint)) .build(); } - public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map> comments) { + public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) { String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); boolean isHint = isHint(type, dossierTemplateId); - return RedactionLogEntry.builder() + return EntityLogEntry.builder() .id(manualEntity.getId()) .color(getColor(type, dossierTemplateId, manualEntity.applied())) .reason(manualEntity.buildReasonWithManualChangeDescriptions()) .legalBasis(manualEntity.legalBasis()) .value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue())) .type(type) - .redacted(manualEntity.applied()) - .isHint(isHint) - .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION)) - .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) + .state(buildEntryState(manualEntity)) + .entryType(buildEntryType(manualEntity, isHint)) .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) .sectionNumber(0) .matchedRule("ManualRedaction") @@ -185,33 +145,29 @@ public class EntityLogCreatorService { .engines(Collections.emptySet()) .reference(Collections.emptySet()) .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint)) - .comments(buildRedactionLogComments(comments, manualEntity.getId())) .build(); } - public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map> comments) { + public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) { String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); - return RedactionLogEntry.builder() + return EntityLogEntry.builder() .id(image.getId()) .color(getColor(imageType, dossierTemplateId, image.applied())) - .isImage(true) .type(imageType) - .redacted(image.applied()) .reason(image.buildReasonWithManualChangeDescriptions()) .legalBasis(image.legalBasis()) .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) - .isHint(isHint) .isDictionaryEntry(false) - .isRecommendation(false) .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber()))) .sectionNumber(image.getTreeId().get(0)) .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) .imageHasTransparency(image.isTransparent()) .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint)) - .comments(buildRedactionLogComments(comments, image.getId())) + .state(buildEntryState(image)) + .entryType(buildEntryType(image, isHint)) .build(); } @@ -232,11 +188,47 @@ public class EntityLogCreatorService { } - private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) { + private EntryState buildEntryState(Entity entity) { - return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())), - (float) rectangle2D.getWidth(), - -(float) rectangle2D.getHeight(), + if (entity.applied()) { + return EntryState.APPLIED; + } else if (entity.skipped()) { + return EntryState.SKIPPED; + } else if (entity.ignored()) { + return EntryState.IGNORED; + } else { + return EntryState.REMOVED; + } + } + + + private EntryType buildEntryType(Entity entity, boolean isHint) { + + if (entity instanceof TextEntity textEntity) { + return getEntryType(isHint, textEntity.getEntityType()); + } else if (entity instanceof ManualEntity manualEntity) { + return getEntryType(isHint, manualEntity.getEntityType()); + } else if (entity instanceof Image) { + return EntryType.IMAGE; + } + throw new UnsupportedOperationException("Entity subclass %s not implemented!"); + } + + + private static EntryType getEntryType(boolean isHint, EntityType entityType) { + + return switch (entityType) { + case ENTITY -> isHint ? EntryType.HINT : EntryType.ENTITY; + case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; + case RECOMMENDATION -> EntryType.RECOMMENDATION; + case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; + }; + } + + + private Position toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) { + + return new Position(new float[]{(float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight()), (float) rectangle2D.getWidth(), -(float) rectangle2D.getHeight()}, pageNumber); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java index fad2488a..19574ce6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java @@ -5,6 +5,9 @@ import java.util.List; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; @@ -25,6 +28,7 @@ public class ImportedRedactionService { private final RedactionStorageService redactionStorageService; + @Deprecated(forRemoval = true) @Timed("redactmanager_processImportedRedactions") public List processImportedRedactions(String dossierTemplateId, String dossierId, @@ -47,6 +51,29 @@ public class ImportedRedactionService { } + @Timed("redactmanager_processImportedRedactions") + public List processImportedEntities(String dossierTemplateId, + String dossierId, + String fileId, + List redactionLogEntries, + boolean addImportedRedactions) { + + var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId); + if (importedRedactions == null) { + return redactionLogEntries; + } + + redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions)); + + if (addImportedRedactions) { + return addImportedRedactionsEntityLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions); + } + + return redactionLogEntries; + } + + + @Deprecated(forRemoval = true) private List addImportedRedactionsRedactionLogEntries(String dossierTemplateId, List redactionLogEntries, ImportedRedactions importedRedactions) { @@ -70,6 +97,40 @@ public class ImportedRedactionService { } + private List addImportedRedactionsEntityLogEntries(String dossierTemplateId, List entityLogEntries, ImportedRedactions importedRedactions) { + + for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) { + for (var importedRedaction : importedRedactionsValues) { + EntityLogEntry redactionLogEntry = EntityLogEntry.builder() + .id(importedRedaction.getId()) + .type(IMPORTED_REDACTION_TYPE) + .imported(true) + .state(EntryState.APPLIED) + .positions(toPositions(importedRedaction.getPositions())) + .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId)) + .build(); + + entityLogEntries.add(redactionLogEntry); + } + } + + return entityLogEntries; + } + + + private List toPositions(List positions) { + + return positions.stream().map(this::toPosition).toList(); + } + + + private Position toPosition(Rectangle rectangle) { + + return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage()); + } + + + @Deprecated(forRemoval = true) private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) { for (Rectangle rectangle : redactionLogEntry.getPositions()) { @@ -91,6 +152,27 @@ public class ImportedRedactionService { } + private void addIntersections(EntityLogEntry redactionLogEntry, ImportedRedactions importedRedactions) { + + for (Position rectangle : redactionLogEntry.getPositions()) { + var normalizedRectangle = normalize(rectangle); + if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) { + var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber()); + for (ImportedRedaction importedRedaction : importedRedactionsOnPage) { + for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) { + if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) { + if (redactionLogEntry.getImportedRedactionIntersections() == null) { + redactionLogEntry.setImportedRedactionIntersections(new HashSet<>()); + } + redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId()); + } + } + } + } + } + } + + boolean valueInRange(float value, float min, float max) { return round(value) >= round(min) && round(value) <= round(max); @@ -111,6 +193,7 @@ public class ImportedRedactionService { } + @Deprecated(forRemoval = true) private Rectangle normalize(Rectangle rectangle) { Rectangle r = new Rectangle(); @@ -138,6 +221,33 @@ public class ImportedRedactionService { } + private Rectangle normalize(Position position) { + + Rectangle r = new Rectangle(); + Point p = new Point(); + + if (position.getRectangle()[2] < 0) { + p.setX(position.getRectangle()[0] + position.getRectangle()[2]); + r.setWidth(Math.abs(position.getRectangle()[2])); + } else { + p.setX(position.getRectangle()[0]); + r.setWidth(position.getRectangle()[2]); + } + + if (position.getRectangle()[3] < 0) { + p.setY(position.getRectangle()[1] + position.getRectangle()[3]); + r.setHeight(Math.abs(position.getRectangle()[3])); + } else { + p.setY(position.getRectangle()[1]); + r.setHeight(position.getRectangle()[3]); + } + + r.setTopLeft(p); + r.setPage(position.getPageNumber()); + return r; + } + + private float round(float value) { double d = Math.pow(10, 0); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java index e240709f..0f03176a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java @@ -12,8 +12,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; @Service public class ManualChangeFactory { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java new file mode 100644 index 00000000..ddcc9dcd --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -0,0 +1,279 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.awt.geom.Rectangle2D; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; +import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +@RequiredArgsConstructor +public class RedactionLogCreatorService { + + private final DictionaryService dictionaryService; + private final ManualChangeFactory manualChangeFactory; + + + public List createRedactionLog(Document document, + String dossierTemplateId, + List notFoundManualRedactionEntries, + Map> comments) { + + List entries = new ArrayList<>(); + document.getEntities() + .stream() + .filter(RedactionLogCreatorService::isEntityOrRecommendationType) + .filter(entity -> !entity.removed()) + .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments))); + document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments))); + notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments))); + return entries; + } + + + private static boolean isEntityOrRecommendationType(TextEntity textEntity) { + + return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION; + } + + + private List toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map> comments) { + + Set processedIds = new HashSet<>(); + List redactionLogEntities = new ArrayList<>(); + + for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { + + // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows. + if (processedIds.contains(positionOnPage.getId())) { + continue; + } + processedIds.add(positionOnPage.getId()); + + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId); + redactionLogEntry.setId(positionOnPage.getId()); + redactionLogEntry.setComments(buildRedactionLogComments(comments, positionOnPage.getId())); + + List rectanglesPerLine = positionOnPage.getRectanglePerLine() + .stream() + .map(rectangle2D -> toRedactionLogRectangle(rectangle2D, positionOnPage.getPage().getNumber())) + .toList(); + + redactionLogEntry.setPositions(rectanglesPerLine); + redactionLogEntities.add(redactionLogEntry); + } + + return redactionLogEntities; + } + + + private List buildRedactionLogComments(Map> commentsPerId, String id) { + + if (!commentsPerId.containsKey(id)) { + return Collections.emptyList(); + } + List comments = commentsPerId.get(id); + if (comments == null || comments.isEmpty()) { + return Collections.emptyList(); + } + return toRedactionLogComments(comments); + } + + + private List toRedactionLogComments(List comments) { + + return comments.stream().map(this::toRedactionLogComment).toList(); + } + + + private RedactionLogComment toRedactionLogComment(Comment comment) { + + return new RedactionLogComment(comment.getId(), + comment.getUser(), + comment.getText(), + comment.getAnnotationId(), + comment.getFileId(), + comment.getDate(), + comment.getSoftDeletedTime()); + } + + + private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) { + + Set referenceIds = new HashSet<>(); + entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); + int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); + boolean isHint = isHint(entity.getType(), dossierTemplateId); + return RedactionLogEntry.builder() + .color(getColor(entity.getType(), dossierTemplateId, entity.applied())) + .reason(entity.buildReasonWithManualChangeDescriptions()) + .legalBasis(entity.legalBasis()) + .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) + .type(entity.getType()) + .redacted(entity.applied()) + .isHint(isHint) + .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION)) + .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) + .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) + .sectionNumber(sectionNumber) + .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) + .isDictionaryEntry(entity.isDictionaryEntry()) + .textAfter(entity.getTextAfter()) + .textBefore(entity.getTextBefore()) + .startOffset(entity.getTextRange().start()) + .endOffset(entity.getTextRange().end()) + .isDossierDictionaryEntry(entity.isDossierDictionaryEntry()) + .engines(getEngines(entity)) + .reference(referenceIds) + .manualChanges(mapManualChanges(entity.getManualOverwrite(), isHint)) + .build(); + } + + + private Set getEngines(TextEntity entity) { + + return entity.getEngines() != null ? mapToEngines(entity.getEngines()) : Collections.emptySet(); + } + + + private Set mapToEngines(Set engines) { + + return engines.stream().map(engine -> switch (engine) { + case NER -> Engine.NER; + case RULE -> Engine.RULE; + case DICTIONARY -> Engine.DICTIONARY; + default -> null; + }).filter(Objects::nonNull).collect(Collectors.toSet()); + } + + + public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map> comments) { + + String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); + boolean isHint = isHint(type, dossierTemplateId); + return RedactionLogEntry.builder() + .id(manualEntity.getId()) + .color(getColor(type, dossierTemplateId, manualEntity.applied())) + .reason(manualEntity.buildReasonWithManualChangeDescriptions()) + .legalBasis(manualEntity.legalBasis()) + .value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue())) + .type(type) + .redacted(manualEntity.applied()) + .isHint(isHint) + .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION)) + .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) + .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) + .sectionNumber(0) + .matchedRule("ManualRedaction") + .rectangle(manualEntity.isRectangle()) + .isDictionaryEntry(manualEntity.isDictionaryEntry()) + .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) + .textAfter("") + .textBefore("") + .startOffset(-1) + .endOffset(-1) + .positions(manualEntity.getEntityPosition() + .stream() + .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .collect(Collectors.toList())) + .engines(Collections.emptySet()) + .reference(Collections.emptySet()) + .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint)) + .comments(buildRedactionLogComments(comments, manualEntity.getId())) + .build(); + } + + + private List mapManualChanges(ManualChangeOverwrite manualEntity, boolean isHint) { + + return manualChangeFactory.toManualChangeList(manualEntity.getManualChangeLog(), isHint).stream().map(this::mapManualChange).toList(); + } + + + private ManualChange mapManualChange(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange manualChange) { + + ManualChange manualChange1 = new ManualChange(); + BeanUtils.copyProperties(manualChange, manualChange1); + return manualChange1; + } + + + public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map> comments) { + + String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); + boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); + return RedactionLogEntry.builder() + .id(image.getId()) + .color(getColor(imageType, dossierTemplateId, image.applied())) + .isImage(true) + .type(imageType) + .redacted(image.applied()) + .reason(image.buildReasonWithManualChangeDescriptions()) + .legalBasis(image.legalBasis()) + .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) + .isHint(isHint) + .isDictionaryEntry(false) + .isRecommendation(false) + .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber()))) + .sectionNumber(image.getTreeId().get(0)) + .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString())) + .imageHasTransparency(image.isTransparent()) + .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint)) + .comments(buildRedactionLogComments(comments, image.getId())) + .build(); + + } + + + private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { + + if (!isRedaction && !isHint(type, dossierTemplateId)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); + } + + + private boolean isHint(String type, String dossierTemplateId) { + + return dictionaryService.isHint(type, dossierTemplateId); + } + + + private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) { + + return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())), + (float) rectangle2D.getWidth(), + -(float) rectangle2D.getHeight(), + pageNumber); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java index 84ffe542..87cadeb3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java @@ -1,7 +1,17 @@ package com.iqser.red.service.redaction.v1.server.service.drools; +import java.util.Collections; +import java.util.List; +import java.util.Set; + import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Component; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; + import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; @@ -13,5 +23,9 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class ComponentDroolsExecutionService { + public List executeRules(Set entities, List addedFileAttributes, List notFoundManualRedactionEntries, List list) { + + return Collections.emptyList(); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java index 56dd52bf..25874736 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java @@ -9,6 +9,8 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.RuleFileType; +import com.iqser.red.service.redaction.v1.model.Rules; import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule; import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources; @@ -24,20 +26,25 @@ public class DroolsSyntaxValidationService { @SneakyThrows - public DroolsSyntaxValidation testRules(String rules) { + public DroolsSyntaxValidation testRules(Rules rules) { - DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules); - DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules); + DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), rules.getRuleFileType()); + DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules.getRulesString()); droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages()); return droolsCompilerSyntaxValidation; } - private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString) { + private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) { RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString); DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation(); - RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileInputStreamString()); + + RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) { + case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileInputStreamString()); + case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseCopmonentRuleFileInputStreamString()); + }; + if (!ruleFileBluePrint.getImports().equals(baseRuleFileBluePrint.getImports())) { customSyntaxValidation.getDroolsSyntaxErrorMessages() .add(DroolsSyntaxErrorMessage.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java index f6d1cc19..57237734 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java @@ -24,6 +24,7 @@ public class KieContainerCreationService { private final RulesClient rulesClient; + public KieWrapper getLatestKieContainer(String dossierTemplateId) { long version = rulesClient.getVersion(dossierTemplateId); @@ -38,6 +39,8 @@ public class KieContainerCreationService { KieServices kieServices = KieServices.Factory.get(); return kieServices.newReleaseId("com.knecon.rules", TenantContext.getTenantId() + ":" + dossierTemplate, String.format("1.%d", version)); } + + public KieContainer getKieContainer(String dossierTemplateId, long version) { KieServices kieServices = KieServices.Factory.get(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/ComponentCreationService.java new file mode 100644 index 00000000..b876b08b --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/ComponentCreationService.java @@ -0,0 +1,5 @@ +package com.iqser.red.service.redaction.v1.server.service.graph; + +public class ComponentCreationService { + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/EntityCreationService.java index 9a4b8bee..70cd2b0c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/graph/EntityCreationService.java @@ -17,7 +17,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.kie.api.runtime.KieSession; import com.google.common.base.Functions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 419960bf..634b1fb0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -6,6 +6,7 @@ import java.io.InputStream; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions; @@ -86,6 +87,18 @@ public class RedactionStorageService { } + @Timed("redactmanager_getRedactionLog") + public EntityLog getEntityLog(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class); + } catch (StorageObjectDoesNotExist e) { + log.debug("RedactionLog not available."); + return null; + } + + } + @Timed("redactmanager_getDocumentGraph") public DocumentData getDocumentData(String dossierId, String fileId) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java index bec5b39b..339776ea 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RuleManagementResources.java @@ -26,4 +26,19 @@ public class RuleManagementResources { } } + @SneakyThrows + public static InputStream getBaseComponentRuleFileInputStream() { + + return new ClassPathResource(Path.of(folderPrefix, "base_component_rules.drl").toString()).getInputStream(); + } + + + @SneakyThrows + public static String getBaseCopmonentRuleFileInputStreamString() { + + try (var in = getBaseRuleFileInputStream()) { + return new String(in.readAllBytes()); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl new file mode 100644 index 00000000..3bbb5636 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl @@ -0,0 +1,89 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.graph.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; +global Document document +global EntityCreationService entityCreationService +global ManualChangesApplicationService manualChangesApplicationService +global Dictionary dictionary + +/** +This file contains all the rules that are in the templates. +The Imports from this file are all that are possible and will be imported for all entity files. +Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution! +*/ + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +//------------------------------------ Local dictionary search rules ------------------------------------ + +// Rule unit: LocalDictionarySearch.0 +rule "LDS.0.0: run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) + .forEach(entity -> { + Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); + entity.addMatchedRules(matchedRules); + }); + end + +// Everything above this comment will be prepended to any entity rule file before execution. diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl index 36aecaf1..fdc9d49d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl @@ -46,13 +46,14 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 77602279..b76299c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -22,7 +22,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 04edd761..82e2c1c9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -503,7 +503,7 @@ public class RulesTest { assertThat(savedManualChange).isPresent(); } - assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(new Engine[0])); + assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(Engine[]::new)); assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference()); assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java index edb684bb..b771e9f9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java @@ -29,7 +29,7 @@ import com.iqser.red.service.redaction.v1.server.service.graph.DocumentGraphMapp import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.graph.ManualEntityCreationService; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService; +import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -42,7 +42,7 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest { @Autowired private ManualEntityCreationService redactionLogAdapter; @Autowired - private EntityLogCreatorService entityLogCreatorService; + private RedactionLogCreatorService redactionLogCreatorService; @BeforeEach @@ -91,7 +91,7 @@ public class MigrationPocTest extends BuildDocumentIntegrationTest { Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(request.getDossierId(), request.getFileId())); List notFoundManualRedactionEntries = redactionLogAdapter.toRedactionEntity(originalRedactionLog, document); - var migratedRedactionLogEntries = entityLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID, notFoundManualRedactionEntries, Collections.emptyMap()); + var migratedRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID, notFoundManualRedactionEntries, Collections.emptyMap()); Map migratedIds = migratedRedactionLogEntries.stream().collect(toMap(RedactionLogEntry::getId, Functions.identity())); Map newIds = newRedactionLog.getRedactionLogEntry().stream().collect(toMap(RedactionLogEntry::getId, Functions.identity())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java index 837e3404..df678676 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java @@ -10,6 +10,8 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.core.io.ClassPathResource; import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation; +import com.iqser.red.service.redaction.v1.model.RuleFileType; +import com.iqser.red.service.redaction.v1.model.Rules; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.service.graph.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService; @@ -41,7 +43,7 @@ class DroolsSyntaxValidationServiceTest { String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString)); droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); assertTrue(droolsSyntaxValidation.isCompiled()); } @@ -56,7 +58,7 @@ class DroolsSyntaxValidationServiceTest { String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString)); droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); assertTrue(droolsSyntaxValidation.isCompiled()); } @@ -71,7 +73,7 @@ class DroolsSyntaxValidationServiceTest { String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString)); droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); assertTrue(droolsSyntaxValidation.isCompiled()); } @@ -86,7 +88,7 @@ class DroolsSyntaxValidationServiceTest { String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString)); droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); assertTrue(droolsSyntaxValidation.isCompiled()); } @@ -101,7 +103,7 @@ class DroolsSyntaxValidationServiceTest { String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString)); droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); assertTrue(droolsSyntaxValidation.isCompiled()); } @@ -116,7 +118,7 @@ class DroolsSyntaxValidationServiceTest { String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(rulesString); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, rulesString)); droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println); assertTrue(droolsSyntaxValidation.isCompiled()); } @@ -132,7 +134,7 @@ class DroolsSyntaxValidationServiceTest { String rulesString = new String(rulesFile.getInputStream().readAllBytes()); String corruptedRules = rulesString.replaceAll(";", ""); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(corruptedRules); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, corruptedRules)); assertFalse(droolsSyntaxValidation.isCompiled()); } @@ -145,10 +147,10 @@ class DroolsSyntaxValidationServiceTest { var rulesFile = new ClassPathResource("drools/rules.drl"); String rulesString = new String(rulesFile.getInputStream().readAllBytes()); - String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.graph.nodes.ImageType;", + String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;", ""); - DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(corruptedRules); + DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new Rules(RuleFileType.ENTITY, corruptedRules)); assertFalse(droolsSyntaxValidation.isCompiled()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java index e44a2ef6..4da12c1d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java @@ -29,7 +29,7 @@ import com.iqser.red.service.redaction.v1.server.service.graph.EntityEnrichmentS import com.iqser.red.service.redaction.v1.server.service.graph.ManualEntityCreationService; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; -import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService; +import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService; import lombok.SneakyThrows; @@ -42,7 +42,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes private ManualEntityCreationService manualEntityCreationService; @Autowired - private EntityLogCreatorService entityLogCreatorService; + private RedactionLogCreatorService redactionLogCreatorService; @MockBean private DictionaryService dictionaryService; @@ -120,7 +120,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes assertEquals(1, notFoundManualEntities.size()); assertTrue(document.getEntities().isEmpty()); - List redactionLogEntries = entityLogCreatorService.createRedactionLog(document, + List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID, notFoundManualEntities, Collections.emptyMap()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 5be6198b..2e18ed82 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index f088c04d..ac0af696 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 92a210bc..23368fb7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index b9a162ea..16ecf53f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index f10d7350..2e7ebdd1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -54,12 +54,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService global Dictionary dictionary - //------------------------------------ queries ------------------------------------ query "getFileAttributes" diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 476ef78a..1aae7dfb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -46,7 +46,7 @@ import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicatio import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -54,7 +54,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index fb71a877..2dbe211c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -25,7 +25,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;