RED-6009 - Document Tree Structure

* implement ManualRedactionEntry Handling

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -151,37 +151,21 @@ public class AnalyzeService {
                 new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
                 analyzeRequest.getDossierId());
 
-        Set<Integer> sectionsToReanalyseIds = analyzeRequest.getSectionsToReanalyse().isEmpty() //
+        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, redactionLog, documentGraph, dictionaryIncrement);
-                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
+        List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(documentGraph, sectionsToReanalyseIds);
-                : analyzeRequest.getSectionsToReanalyse();
+        log.info("Should reanalyze {} sections for request: {}", sectionsToReAnalyse.size(), analyzeRequest);
-        log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyseIds.size(), analyzeRequest);
 
-        if (sectionsToReanalyseIds.isEmpty()) {
+        if (sectionsToReAnalyse.isEmpty()) {
             return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
         }
 
-        List<SemanticNode> sectionsToReAnalyse = documentGraph.streamChildren()
+        List<EntityRecognitionEntity> nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
-                .filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0)))
-                .collect(Collectors.toList());
-
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        sectionsToReAnalyse.forEach(sectionNode -> entityRedactionService.addDictionaryEntities(dictionary, sectionNode));
-
-        List<EntityRecognitionEntity> nerEntities;
-        if (redactionServiceSettings.isNerServiceEnabled()) {
-            nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph);
-        } else {
-            nerEntities = Collections.emptyList();
-        }
-
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
 
-        Set<FileAttribute> addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary,
+        sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
-                documentGraph,
+        entityRedactionService.applyManualRedactionEntries(analyzeRequest.getManualRedactions(), documentGraph);
-                sectionsToReAnalyse,
+        Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
-                kieContainer,
-                analyzeRequest,
-                nerEntities);
 
         List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
 
@@ -198,11 +182,21 @@ public class AnalyzeService {
     }
 
 
-    @Timed("redactmanager_analyze")
+    private static List<SemanticNode> getSectionsToReAnalyse(DocumentGraph documentGraph, Set<Integer> sectionsToReanalyseIds) {
-    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
 
-        long startTime = System.currentTimeMillis();
+        return documentGraph.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))).collect(Collectors.toList());
-        DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+    }
+
+
+    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DocumentGraph documentGraph, DictionaryIncrement dictionaryIncrement) {
+
+        return analyzeRequest.getSectionsToReanalyse().isEmpty() //
+                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
+                : analyzeRequest.getSectionsToReanalyse();
+    }
+
+
+    private List<EntityRecognitionEntity> getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
 
         List<EntityRecognitionEntity> nerEntities;
         if (redactionServiceSettings.isNerServiceEnabled()) {
@@ -210,13 +204,28 @@ public class AnalyzeService {
         } else {
             nerEntities = Collections.emptyList();
         }
+        return nerEntities;
+    }
+
+
+    @Timed("redactmanager_analyze")
+    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+        DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+
+        List<EntityRecognitionEntity> nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
 
         dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
         long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
 
-        Set<FileAttribute> addedFileAttributes = entityRedactionService.addDictionaryAndRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities);
+        log.debug("Starting Dictionary Search");
+        long dictSearchStart = System.currentTimeMillis();
+        entityRedactionService.addDictionaryEntities(dictionary, documentGraph);
+        log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
+        Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities);
 
         List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -15,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@@ -63,7 +65,7 @@ class SectionFinder {
                 sectionsToReanalyse.add(mainNode.getTocId().get(0));
             }
         });
-        
+
         log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
 
         return sectionsToReanalyse;
@@ -80,7 +82,11 @@ class SectionFinder {
                 Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
                         Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
                                 Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+                                        Stream.concat(manualRedactions.getEntriesToAdd()
+                                                        .stream()
+                                                        .map(ManualRedactionEntry::getSourceId)
+                                                        .filter(sourceId -> !Objects.isNull(sourceId)),
+                                                manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))))).collect(Collectors.toSet());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java

@@ -1,6 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
 
+import java.util.Collection;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -9,13 +11,18 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
@@ -34,18 +41,14 @@ public class EntityRedactionService {
 
     DroolsExecutionService droolsExecutionService;
     EntityCreationService entityCreationService;
+    ManualRedactionApplicationService manualRedactionApplicationService;
 
 
-    public Set<FileAttribute> addDictionaryAndRuleEntities(Dictionary dictionary,
+    public Set<FileAttribute> addRuleEntities(Dictionary dictionary,
-                                                           DocumentGraph documentGraph,
+                                              DocumentGraph documentGraph,
-                                                           KieContainer kieContainer,
+                                              KieContainer kieContainer,
-                                                           AnalyzeRequest analyzeRequest,
+                                              AnalyzeRequest analyzeRequest,
-                                                           List<EntityRecognitionEntity> nerEntities) {
+                                              List<EntityRecognitionEntity> nerEntities) {
-
-        log.debug("Starting Dictionary Search");
-        long dictSearchStart = System.currentTimeMillis();
-        addDictionaryEntities(dictionary, documentGraph);
-        log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
 
         log.debug("Starting Drools Execution");
         long droolsStart = System.currentTimeMillis();
@@ -60,17 +63,12 @@ public class EntityRedactionService {
     }
 
 
-    public Set<FileAttribute> addDictionaryAndRuleEntities(Dictionary dictionary,
+    public Set<FileAttribute> addRuleEntities(Dictionary dictionary,
-                                                           DocumentGraph documentGraph,
+                                              DocumentGraph documentGraph,
-                                                           List<SemanticNode> sectionsToReanalyze,
+                                              List<SemanticNode> sectionsToReanalyze,
-                                                           KieContainer kieContainer,
+                                              KieContainer kieContainer,
-                                                           AnalyzeRequest analyzeRequest,
+                                              AnalyzeRequest analyzeRequest,
-                                                           List<EntityRecognitionEntity> nerEntities) {
+                                              List<EntityRecognitionEntity> nerEntities) {
-
-        log.debug("Starting Dictionary Search");
-        long dictSearchStart = System.currentTimeMillis();
-        sectionsToReanalyze.forEach(node -> addDictionaryEntities(dictionary, node));
-        log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
 
         log.debug("Starting Drools execution");
         long ruleExecutionStart = System.currentTimeMillis();
@@ -87,6 +85,53 @@ public class EntityRedactionService {
     }
 
 
+    public void applyManualRedactionEntries(ManualRedactions manualRedactions, DocumentGraph documentGraph) {
+
+        if (manualRedactions == null) {
+            return;
+        }
+        manualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(manualRedactionEntry -> Objects.nonNull(manualRedactionEntry.getSourceId()))
+                .forEach(manualRedactionEntry -> applyManualRedactionEntry(manualRedactionEntry, documentGraph));
+
+    }
+
+
+    private void applyManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, DocumentGraph documentGraph) {
+
+        EntityType entityType = parseEntityType(manualRedactionEntry.getReason());
+        documentGraph.getPages()
+                .stream()
+                .filter(page -> isInManualRedactionEntry(manualRedactionEntry, page))
+                .map(PageNode::getEntities)
+                .flatMap(Collection::stream)
+                .filter(entity -> entity.matchesAnnotationId(manualRedactionEntry.getSourceId()))
+                .filter(entity -> entity.getValue().equals(manualRedactionEntry.getValue()))
+                .map(RedactionEntity::getBoundary)
+                .map(bound -> entityCreationService.byBoundary(bound, manualRedactionEntry.getType(), entityType, documentGraph))
+                .peek(entity -> entity.setLegalBasis(manualRedactionEntry.getLegalBasis()))
+                .forEach(entity -> entity.setRedactionReason(manualRedactionEntry.getReason()));
+    }
+
+
+    private EntityType parseEntityType(String reason) {
+
+        return switch (reason.toLowerCase()) {
+            case "false positive" -> EntityType.FALSE_POSITIVE;
+            case "false recommendation" -> EntityType.FALSE_RECOMMENDATION;
+            case "recommendation" -> EntityType.RECOMMENDATION;
+            default -> EntityType.ENTITY;
+        };
+    }
+
+
+    private static boolean isInManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, PageNode page) {
+
+        return manualRedactionEntry.getPositions().stream().map(Rectangle::getPage).collect(Collectors.toSet()).contains(page.getNumber());
+    }
+
+
     public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
 
         for (var model : dictionary.getDictionaryModels()) {