RED-6009 - Document Tree Structure

* removed ner entity validation due to poor performance

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java

@@ -18,7 +18,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.te
 
 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
 public class NerEntitiesAdapter {
@@ -33,20 +35,18 @@ public class NerEntitiesAdapter {
 
 
     /**
-     * Adds the appropriate Offsets to the entities from the NER Service and validates, that the values match.
+     * Adds the appropriate Offsets to the entities from the NER Service.
      *
      * @param nerEntitiesModel the Entities just as the NER Service returns them
      * @param documentGraph    the document structure, from which the NER Service found the entities
      * @return a stream of validated entities
      */
-    public List<NerEntities.NerEntity> getValidatedEntities(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
+    public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
 
-        return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntitiesModel) //
+        return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(
-                .filter(nerEntityModel -> nerEntityOffsetMatchesDocumentGraphOffsets(nerEntityModel, documentGraph.buildTextBlock()))
+                nerEntityModel.getValue(),
-                .map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
+                new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
-                        new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
+                nerEntityModel.getType())).toList());
-                        nerEntityModel.getType()))
-                .toList();
     }
 
 
@@ -150,12 +150,6 @@ public class NerEntitiesAdapter {
     }
 
 
-    private boolean nerEntityOffsetMatchesDocumentGraphOffsets(EntityRecognitionEntity nerEntity, TextBlock textBlock) {
-
-        return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset()));
-    }
-
-
     private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) {
 
         nerEntitiesModel.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -160,7 +160,7 @@ public class AnalyzeService {
 
         Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, redactionLog, documentGraph, dictionaryIncrement);
         List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(documentGraph, sectionsToReanalyseIds);
-        log.info("Should reanalyze {} sections for request: {}", sectionsToReAnalyse.size(), analyzeRequest);
+        log.info("Should reanalyze {} sections", sectionsToReAnalyse.size());
 
         if (sectionsToReAnalyse.isEmpty()) {
             return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
@@ -213,7 +213,7 @@ public class AnalyzeService {
         if (redactionServiceSettings.isNerServiceEnabled()) {
             NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
             log.info("Get NerEntitiesModel");
-            nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(nerEntitiesModel, documentGraph));
+            nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
         } else {
             nerEntities = new NerEntities(Collections.emptyList());
         }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java

@@ -19,6 +19,7 @@ import org.springframework.core.io.ClassPathResource;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentGraphIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
@@ -27,7 +28,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
 
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
 
     @Autowired
@@ -57,7 +60,8 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
 
         ClassPathResource resource = new ClassPathResource(filePath);
         PDDocument pdDocument = PDDocument.load(resource.getInputStream());
-        Stream<NerEntities.NerEntity> unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
+        Stream<NerEntities.NerEntity> unchangedAddressParts = nerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
+                .getNerEntityList()
                 .stream()
                 .filter(e -> !e.type().equals("CBI_author"));
         List<RedactionEntity> redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts)
@@ -78,9 +82,51 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    public void testGetNerEntitiesLargeFile() {
+
+        String filePath = "files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf";
+        String nerEntitiesFilePath = "ner_entities/S-Metolachlor_RAR_02_Volume_2_2018-09-06.NER_ENTITIES.json";
+        DocumentGraph documentGraph = buildGraphNoImages(filePath);
+        log.info("Graph built");
+        NerEntitiesModel nerEntitiesModel = parseNerEntities(nerEntitiesFilePath);
+        log.info("Parsed NerEntitiesModel");
+        NerEntities nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
+        log.info("Validated and mapped");
+        List<Boundary> nerEntityBoundaries = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities).toList();
+        log.info("Combined to CBI_address");
+        List<RedactionEntity> cbiAddressEntities = nerEntityBoundaries.stream()
+                .map(b -> entityCreationService.byBoundary(b, "CBI_address", EntityType.RECOMMENDATION, documentGraph))
+                .toList();
+        assertFalse(cbiAddressEntities.isEmpty());
+        assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getBoundary().start() < entity.getBoundary().end()));
+
+        ClassPathResource resource = new ClassPathResource(filePath);
+        PDDocument pdDocument = PDDocument.load(resource.getInputStream());
+        List<RedactionEntity> validatedEntities = nerEntitiesAdapter.toNerEntities(parseNerEntities(nerEntitiesFilePath), documentGraph)
+                .getNerEntityList()
+                .stream()
+                .map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, documentGraph))
+                .toList();
+        Stream.concat(cbiAddressEntities.stream(), validatedEntities.stream())
+                .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber()))
+                .forEach((pageNumber, entities) -> drawNerEntitiesAsPartsAndCombined(pageNumber,
+                        getPositionsFromEntityOfType("CBI_author", entities),
+                        getPositionsFromEntityNotOfType(List.of("CBI_author", "CBI_address"), entities),
+                        getPositionsFromEntityOfType("CBI_address", entities),
+                        pdDocument));
+
+        File outputFile = new File("/tmp/nerEntities.pdf");
+        pdDocument.save(outputFile);
+        pdDocument.close();
+
+    }
+
+
     private List<NerEntities.NerEntity> validateAndCombine(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) {
 
-        NerEntities nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(nerEntitiesModel, documentGraph));
+        NerEntities nerEntities = nerEntitiesAdapter.toNerEntities(nerEntitiesModel, documentGraph);
 
         List<NerEntities.NerEntity> cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList();
         Stream<NerEntities.NerEntity> cbiAddress = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)