RED-6009 - Document Tree Structure

* removed ner entity validation due to poor performance
* filtered ner entities during reanalysis to improve performance

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java

@@ -18,6 +18,7 @@ import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
@@ -180,7 +181,9 @@ public class EntityCreationService {
 
     public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        return byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
+        RedactionEntity entity = byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
+        entity.addEngine(Engine.NER);
+        return entity;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java

@@ -18,7 +18,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.te
 
 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
 public class NerEntitiesAdapter {
@@ -33,20 +35,18 @@ public class NerEntitiesAdapter {
 
 
     /**
-     * Adds the appropriate Offsets to the entities from the NER Service and validates, that the values match.
+     * Adds the appropriate Offsets to the entities from the NER Service.
      *
      * @param nerEntitiesModel the Entities just as the NER Service returns them
      * @param documentGraph    the document structure, from which the NER Service found the entities
      * @return a stream of validated entities
      */
-    public List<NerEntities.NerEntity> getValidatedEntities(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
+    public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
 
-        return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntitiesModel) //
+        return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(
-                .filter(nerEntityModel -> nerEntityOffsetMatchesDocumentGraphOffsets(nerEntityModel, documentGraph.buildTextBlock()))
+                nerEntityModel.getValue(),
-                .map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
+                new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
-                        new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
+                nerEntityModel.getType())).toList());
-                        nerEntityModel.getType()))
-                .toList();
     }
 
 
@@ -150,12 +150,6 @@ public class NerEntitiesAdapter {
     }
 
 
-    private boolean nerEntityOffsetMatchesDocumentGraphOffsets(EntityRecognitionEntity nerEntity, TextBlock textBlock) {
-
-        return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset()));
-    }
-
-
     private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) {
 
         nerEntitiesModel.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -33,7 +33,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedSectionText;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedText;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
@@ -135,92 +134,6 @@ public class AnalyzeService {
     }
 
 
-    @Timed("redactmanager_reanalyze")
-    @SneakyThrows
-    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
-
-        long startTime = System.currentTimeMillis();
-        log.info("Start reanalysis for request: {}", analyzeRequest);
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        log.info("loaded previous redaction log");
-        DocumentData documentData = redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        log.info("loaded document data");
-        DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(documentData);
-        log.info("constructed graph");
-
-        // not yet ready for reanalysis
-        if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) {
-            return analyze(analyzeRequest);
-        }
-
-        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
-                new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
-                analyzeRequest.getDossierId());
-        log.info("dictionary increment loaded {}", dictionaryIncrement);
-
-        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, redactionLog, documentGraph, dictionaryIncrement);
-        List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(documentGraph, sectionsToReanalyseIds);
-        log.info("Should reanalyze {} sections for request: {}", sectionsToReAnalyse.size(), analyzeRequest);
-
-        if (sectionsToReAnalyse.isEmpty()) {
-            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
-        }
-
-        NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph);
-        log.info("Found {} Entity Recognition Service Entities", nerEntities.getNerEntityList().size());
-        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
-        log.info("Rules updated");
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        log.info("Dictionary updated");
-
-        sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
-        log.info("Dictionary search finished");
-        Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
-        log.info("Rule execution finished");
-
-        List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
-
-        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                newRedactionLogEntries,
-                false);
-
-        redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
-        redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
-
-        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, addedFileAttributes);
-    }
-
-
-    private static List<SemanticNode> getSectionsToReAnalyse(DocumentGraph documentGraph, Set<Integer> sectionsToReanalyseIds) {
-
-        return documentGraph.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))).collect(Collectors.toList());
-    }
-
-
-    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DocumentGraph documentGraph, DictionaryIncrement dictionaryIncrement) {
-
-        return analyzeRequest.getSectionsToReanalyse().isEmpty() //
-                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
-                : analyzeRequest.getSectionsToReanalyse();
-    }
-
-
-    private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
-
-        NerEntities nerEntities;
-        if (redactionServiceSettings.isNerServiceEnabled()) {
-            NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-            log.info("Get NerEntitiesModel");
-            nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(nerEntitiesModel, documentGraph));
-        } else {
-            nerEntities = new NerEntities(Collections.emptyList());
-        }
-        return nerEntities;
-    }
-
-
     @Timed("redactmanager_analyze")
     public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
 
@@ -263,6 +176,55 @@ public class AnalyzeService {
     }
 
 
+    @Timed("redactmanager_reanalyze")
+    @SneakyThrows
+    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+
+        // not yet ready for reanalysis
+        if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) {
+            return analyze(analyzeRequest);
+        }
+
+        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
+                new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
+                analyzeRequest.getDossierId());
+
+        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, redactionLog, documentGraph, dictionaryIncrement);
+        List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(documentGraph, sectionsToReanalyseIds);
+
+        if (sectionsToReAnalyse.isEmpty()) {
+            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
+        }
+
+        NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, documentGraph, sectionsToReanalyseIds);
+        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
+        sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
+
+        long ruleStart = System.currentTimeMillis();
+        Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
+        log.info("Rule execution took {} ms", System.currentTimeMillis() - ruleStart);
+
+        List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId());
+
+        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                newRedactionLogEntries,
+                false);
+
+        redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
+        redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
+
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, addedFileAttributes);
+    }
+
+
     private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                            long startTime,
                                            RedactionLog redactionLog,
@@ -305,6 +267,54 @@ public class AnalyzeService {
     }
 
 
+    private static List<SemanticNode> getSectionsToReAnalyse(DocumentGraph documentGraph, Set<Integer> sectionsToReanalyseIds) {
+
+        return documentGraph.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))).collect(Collectors.toList());
+    }
+
+
+    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DocumentGraph documentGraph, DictionaryIncrement dictionaryIncrement) {
+
+        return analyzeRequest.getSectionsToReanalyse().isEmpty() //
+                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
+                : analyzeRequest.getSectionsToReanalyse();
+    }
+
+
+    private NerEntities getEntityRecognitionEntitiesFilteredBySectionIds(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph, Set<Integer> sectionsToReanalyseIds) {
+
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
+            NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+            nerEntitiesModel = filterNerEntitiesModelBySectionIds(sectionsToReanalyseIds, nerEntitiesModel);
+            nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
+        } else {
+            nerEntities = new NerEntities(Collections.emptyList());
+        }
+        return nerEntities;
+    }
+
+
+    private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set<Integer> sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) {
+
+        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet().stream() //
+                .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+    }
+
+
+    private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) {
+
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
+            nerEntities = nerEntitiesAdapter.toNerEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph);
+        } else {
+            nerEntities = new NerEntities(Collections.emptyList());
+        }
+        return nerEntities;
+    }
+
+
     public List<RedactionLogLegalBasis> toSimplifiedSectionText(List<LegalBasis> legalBasis) {
 
         return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java

@@ -19,6 +19,7 @@ import org.springframework.core.io.ClassPathResource;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentGraphIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
@@ -27,7 +28,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
 
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
 
     @Autowired
@@ -57,7 +60,8 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
 
         ClassPathResource resource = new ClassPathResource(filePath);
         PDDocument pdDocument = PDDocument.load(resource.getInputStream());
-        Stream<NerEntities.NerEntity> unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
+        Stream<NerEntities.NerEntity> unchangedAddressParts = nerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
+                .getNerEntityList()
                 .stream()
                 .filter(e -> !e.type().equals("CBI_author"));
         List<RedactionEntity> redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts)
@@ -78,9 +82,51 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    public void testGetNerEntitiesLargeFile() {
+
+        String filePath = "files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf";
+        String nerEntitiesFilePath = "ner_entities/S-Metolachlor_RAR_02_Volume_2_2018-09-06.NER_ENTITIES.json";
+        DocumentGraph documentGraph = buildGraphNoImages(filePath);
+        log.info("Graph built");
+        NerEntitiesModel nerEntitiesModel = parseNerEntities(nerEntitiesFilePath);
+        log.info("Parsed NerEntitiesModel");
+        NerEntities nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
+        log.info("Validated and mapped");
+        List<Boundary> nerEntityBoundaries = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).toList();
+        log.info("Combined to CBI_address");
+        List<RedactionEntity> cbiAddressEntities = nerEntityBoundaries.stream()
+                .map(b -> entityCreationService.byBoundary(b, "CBI_address", EntityType.RECOMMENDATION, documentGraph))
+                .toList();
+        assertFalse(cbiAddressEntities.isEmpty());
+        assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getBoundary().start() < entity.getBoundary().end()));
+
+        ClassPathResource resource = new ClassPathResource(filePath);
+        PDDocument pdDocument = PDDocument.load(resource.getInputStream());
+        List<RedactionEntity> validatedEntities = nerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
+                .getNerEntityList()
+                .stream()
+                .map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, documentGraph))
+                .toList();
+        Stream.concat(cbiAddressEntities.stream(), validatedEntities.stream())
+                .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber()))
+                .forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber,
+                        getPositionsFromEntityOfType("CBI_author", entities),
+                        getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities),
+                        getPositionsFromEntityOfType("CBI_address", entities),
+                        pdDocument));
+
+        File outputFile = new File("/tmp/nerEntities.pdf");
+        pdDocument.save(outputFile);
+        pdDocument.close();
+
+    }
+
+
     private List<NerEntities.NerEntity> validateAndCombine(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
 
-        NerEntities nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(nerEntitiesModel, documentGraph));
+        NerEntities nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
 
         List<NerEntities.NerEntity> cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList();
         Stream<NerEntities.NerEntity> cbiAddress = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -666,10 +666,10 @@ rule "remove Entity of lower rank, when equal boundaries and entityType"
 rule "remove duplicate FileAttributes"
     salience 64
     when
-        $first: FileAttribute($label: label, $value: value)
+        $fileAttribute: FileAttribute($label: label, $value: value)
-        $second: FileAttribute(this != $first, label == $label, value == $value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
     then
-        retract($second);
+        retract($duplicate);
     end
 
 // --------------------------------------- local dictionary search -------------------------------------------------------------------