From 40307b7e628456f938d7aca44fd41ce6194e7f57 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 3 May 2023 00:06:47 +0200 Subject: [PATCH] RED-6009 - Document Tree Structure * implement ManualRedactionEntry Handling --- .../ManualRedactionApplicationService.java | 42 +++++++++++++++++++ .../service/analyze/AnalyzeService.java | 1 - .../service/analyze/SectionFinder.java | 10 ++++- .../EntityRedactionService.java | 8 ++++ 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java index 0af0ad40..5da6d780 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java @@ -1,14 +1,21 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services; +import java.util.Collection; import java.util.NoSuchElementException; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import lombok.RequiredArgsConstructor; @@ -46,4 +53,39 @@ public class ManualRedactionApplicationService { entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto); } + + public void applyManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, DocumentGraph documentGraph) { + + EntityType entityType = parseEntityType(manualRedactionEntry.getReason()); + documentGraph.getPages() + .stream() + .filter(page -> isInManualRedactionEntry(manualRedactionEntry, page)) + .map(PageNode::getEntities) + .flatMap(Collection::stream) + .filter(entity -> entity.matchesAnnotationId(manualRedactionEntry.getSourceId())) + .filter(entity -> entity.getValue().equals(manualRedactionEntry.getValue())) + .map(RedactionEntity::getBoundary) + .map(bound -> entityCreationService.byBoundary(bound, manualRedactionEntry.getType(), entityType, documentGraph)) + .peek(entity -> entity.setLegalBasis(manualRedactionEntry.getLegalBasis())) + .forEach(entity -> entity.setRedactionReason(manualRedactionEntry.getReason())); + + } + + + private EntityType parseEntityType(String reason) { + + return switch (reason.toLowerCase()) { + case "false positive" -> EntityType.FALSE_POSITIVE; + case "false recommendation" -> EntityType.FALSE_RECOMMENDATION; + case "recommendation" -> EntityType.RECOMMENDATION; + default -> EntityType.ENTITY; + }; + } + + + private static boolean isInManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, PageNode page) { + + return manualRedactionEntry.getPositions().stream().map(Rectangle::getPage).collect(Collectors.toSet()).contains(page.getNumber()); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index dab4c916..f884c74a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -173,7 +173,6 @@ public class AnalyzeService { } else { nerEntities = Collections.emptyList(); } - KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); Set addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java index aa4d0856..82bbd9cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -15,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; @@ -63,7 +65,7 @@ class SectionFinder { sectionsToReanalyse.add(mainNode.getTocId().get(0)); } }); - + log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start); return sectionsToReanalyse; @@ -80,7 +82,11 @@ class SectionFinder { Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId), - manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet()); + Stream.concat(manualRedactions.getEntriesToAdd() + .stream() + .map(ManualRedactionEntry::getSourceId) + .filter(sourceId -> !Objects.isNull(sourceId)), + manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))))).collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index a76e417c..373db2a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; @@ -34,6 +35,7 @@ public class EntityRedactionService { DroolsExecutionService droolsExecutionService; EntityCreationService entityCreationService; + ManualRedactionApplicationService manualRedactionApplicationService; public Set addDictionaryAndRuleEntities(Dictionary dictionary, @@ -72,6 +74,12 @@ public class EntityRedactionService { sectionsToReanalyze.forEach(node -> addDictionaryEntities(dictionary, node)); log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart); + if (analyzeRequest.getManualRedactions() != null) { + analyzeRequest.getManualRedactions() + .getEntriesToAdd() + .forEach(manualRedactionEntry -> manualRedactionApplicationService.applyManualRedactionEntries(manualRedactionEntry, documentGraph)); + } + log.debug("Starting Drools execution"); long ruleExecutionStart = System.currentTimeMillis(); List allFileAttributes = droolsExecutionService.executeRules(kieContainer,