diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index dab4c916..393d1160 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -151,37 +151,21 @@ public class AnalyzeService { new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set sectionsToReanalyseIds = analyzeRequest.getSectionsToReanalyse().isEmpty() // - ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) // - : analyzeRequest.getSectionsToReanalyse(); - log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyseIds.size(), analyzeRequest); + Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, redactionLog, documentGraph, dictionaryIncrement); + List sectionsToReAnalyse = getSectionsToReAnalyse(documentGraph, sectionsToReanalyseIds); + log.info("Should reanalyze {} sections for request: {}", sectionsToReAnalyse.size(), analyzeRequest); - if (sectionsToReanalyseIds.isEmpty()) { + if (sectionsToReAnalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>()); } - List sectionsToReAnalyse = documentGraph.streamChildren() - .filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))) - .collect(Collectors.toList()); - - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - sectionsToReAnalyse.forEach(sectionNode -> entityRedactionService.addDictionaryEntities(dictionary, sectionNode)); - - List nerEntities; - if (redactionServiceSettings.isNerServiceEnabled()) { - nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph); - } else { - nerEntities = Collections.emptyList(); - } - + List nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - Set addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary, - documentGraph, - sectionsToReAnalyse, - kieContainer, - analyzeRequest, - nerEntities); + sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); + entityRedactionService.applyManualRedactionEntries(analyzeRequest.getManualRedactions(), documentGraph); + Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId()); @@ -198,11 +182,21 @@ public class AnalyzeService { } - @Timed("redactmanager_analyze") - public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { + private static List getSectionsToReAnalyse(DocumentGraph documentGraph, Set sectionsToReanalyseIds) { - long startTime = System.currentTimeMillis(); - DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + return documentGraph.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))).collect(Collectors.toList()); + } + + + private Set getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DocumentGraph documentGraph, DictionaryIncrement dictionaryIncrement) { + + return analyzeRequest.getSectionsToReanalyse().isEmpty() // + ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) // + : analyzeRequest.getSectionsToReanalyse(); + } + + + private List getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) { List nerEntities; if (redactionServiceSettings.isNerServiceEnabled()) { @@ -210,13 +204,28 @@ public class AnalyzeService { } else { nerEntities = Collections.emptyList(); } + return nerEntities; + } + + + @Timed("redactmanager_analyze") + public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { + + long startTime = System.currentTimeMillis(); + DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + + List nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph); dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - Set addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities); + log.debug("Starting Dictionary Search"); + long dictSearchStart = System.currentTimeMillis(); + entityRedactionService.addDictionaryEntities(dictionary, documentGraph); + log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart); + Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities); List redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java index aa4d0856..82bbd9cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -15,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; @@ -63,7 +65,7 @@ class SectionFinder { sectionsToReanalyse.add(mainNode.getTocId().get(0)); } }); - + log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start); return sectionsToReanalyse; @@ -80,7 +82,11 @@ class SectionFinder { Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId), - manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet()); + Stream.concat(manualRedactions.getEntriesToAdd() + .stream() + .map(ManualRedactionEntry::getSourceId) + .filter(sourceId -> !Objects.isNull(sourceId)), + manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))))).collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index a76e417c..68567d93 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -1,6 +1,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction; +import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -9,13 +11,18 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; @@ -34,18 +41,14 @@ public class EntityRedactionService { DroolsExecutionService droolsExecutionService; EntityCreationService entityCreationService; + ManualRedactionApplicationService manualRedactionApplicationService; - public Set addDictionaryAndRuleEntities(Dictionary dictionary, - DocumentGraph documentGraph, - KieContainer kieContainer, - AnalyzeRequest analyzeRequest, - List nerEntities) { - - log.debug("Starting Dictionary Search"); - long dictSearchStart = System.currentTimeMillis(); - addDictionaryEntities(dictionary, documentGraph); - log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart); + public Set addRuleEntities(Dictionary dictionary, + DocumentGraph documentGraph, + KieContainer kieContainer, + AnalyzeRequest analyzeRequest, + List nerEntities) { log.debug("Starting Drools Execution"); long droolsStart = System.currentTimeMillis(); @@ -60,17 +63,12 @@ public class EntityRedactionService { } - public Set addDictionaryAndRuleEntities(Dictionary dictionary, - DocumentGraph documentGraph, - List sectionsToReanalyze, - KieContainer kieContainer, - AnalyzeRequest analyzeRequest, - List nerEntities) { - - log.debug("Starting Dictionary Search"); - long dictSearchStart = System.currentTimeMillis(); - sectionsToReanalyze.forEach(node -> addDictionaryEntities(dictionary, node)); - log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart); + public Set addRuleEntities(Dictionary dictionary, + DocumentGraph documentGraph, + List sectionsToReanalyze, + KieContainer kieContainer, + AnalyzeRequest analyzeRequest, + List nerEntities) { log.debug("Starting Drools execution"); long ruleExecutionStart = System.currentTimeMillis(); @@ -87,6 +85,53 @@ public class EntityRedactionService { } + public void applyManualRedactionEntries(ManualRedactions manualRedactions, DocumentGraph documentGraph) { + + if (manualRedactions == null) { + return; + } + manualRedactions.getEntriesToAdd() + .stream() + .filter(manualRedactionEntry -> Objects.nonNull(manualRedactionEntry.getSourceId())) + .forEach(manualRedactionEntry -> applyManualRedactionEntry(manualRedactionEntry, documentGraph)); + + } + + + private void applyManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, DocumentGraph documentGraph) { + + EntityType entityType = parseEntityType(manualRedactionEntry.getReason()); + documentGraph.getPages() + .stream() + .filter(page -> isInManualRedactionEntry(manualRedactionEntry, page)) + .map(PageNode::getEntities) + .flatMap(Collection::stream) + .filter(entity -> entity.matchesAnnotationId(manualRedactionEntry.getSourceId())) + .filter(entity -> entity.getValue().equals(manualRedactionEntry.getValue())) + .map(RedactionEntity::getBoundary) + .map(bound -> entityCreationService.byBoundary(bound, manualRedactionEntry.getType(), entityType, documentGraph)) + .peek(entity -> entity.setLegalBasis(manualRedactionEntry.getLegalBasis())) + .forEach(entity -> entity.setRedactionReason(manualRedactionEntry.getReason())); + } + + + private EntityType parseEntityType(String reason) { + + return switch (reason.toLowerCase()) { + case "false positive" -> EntityType.FALSE_POSITIVE; + case "false recommendation" -> EntityType.FALSE_RECOMMENDATION; + case "recommendation" -> EntityType.RECOMMENDATION; + default -> EntityType.ENTITY; + }; + } + + + private static boolean isInManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, PageNode page) { + + return manualRedactionEntry.getPositions().stream().map(Rectangle::getPage).collect(Collectors.toSet()).contains(page.getNumber()); + } + + public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { for (var model : dictionary.getDictionaryModels()) {