Merge branch 'RED-9206' into 'master'

RED-9206 - Sections are no longer correctly separated from each other in the test file

Closes RED-9206

See merge request redactmanager/redaction-service!415

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl

@@ -1454,6 +1454,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -85,7 +85,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
 
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
                                                                                                        .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
                                                                                                        .type(DOSSIER_REDACTIONS_INDICATOR)
@@ -170,6 +170,60 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testLocalPublishedRemovalWithEntityContainedByEntityOfSameType() throws IOException {
+
+        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
+        AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        dictionary.get(PUBLISHED_INFORMATION_INDICATOR).add("Press");
+        reanlysisVersions.put("Press", 2L);
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
+        mockDictionaryCalls(0L);
+
+        analyzeService.reanalyze(request);
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
+                .orElseThrow();
+        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
+                .orElseThrow();
+        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
+
+        var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
+
+        var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build();
+        request.setManualRedactions(manualRedactions);
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
+                .orElseThrow();
+        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst()
+                .orElseThrow();
+
+        assertEquals(EntryState.APPLIED, asyaLyon2.getState());
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/AcceptanceTest.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+
     private Stream<EntityLogEntry> findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, List<Integer> sectionNumber) {
 
         return redactionLog.getEntityLogEntry()

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -188,6 +188,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                              .addToAllDossiers(false)
                              .updateDictionary(false)
                              .requestDate(OffsetDateTime.now())
+                             .user("user")
                              .build());
         request.setManualRedactions(manualRedactions);
         analyzeService.reanalyze(request);
@@ -233,17 +234,19 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         System.out.println("testManualRedaction");
         long start = System.currentTimeMillis();
-        String pdfFile = "files/syngenta/CustomeFiles/SinglePages/Single Table - Page87_S-Metolachlor_RAR_07_Volume_3CA_B-5_2018-09-06.pdf";
+        String pdfFile = "files/syngenta/CustomerFiles/SinglePages/Single Table - Page87_S-Metolachlor_RAR_07_Volume_3CA_B-5_2018-09-06.pdf";
 
         ManualRedactions manualRedactions = new ManualRedactions();
 
         String manualAddId = UUID.randomUUID().toString();
 
-        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
+        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").user("user").requestDate(OffsetDateTime.now()).build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
                                                            .annotationId("675eba69b0c2917de55462c817adaa05")
                                                            .fileId("fileId")
                                                            .legalBasis("Something")
+                                                           .user("user")
+                                                           .requestDate(OffsetDateTime.now())
                                                            .build()));
 
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
@@ -262,7 +265,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         AnalyzeResult result = analyzeService.analyze(request);
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
-        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
+        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").user("user").build()));
         manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
                                                               .annotationId("675eba69b0c2917de55462c817adaa05")
                                                               .fileId("fileId")
@@ -361,6 +364,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .type("vertebrate")
                 .annotationId(oxfordUniversityPress.getId())
                 .fileId(TEST_FILE_ID)
+                .user("user")
                 .build();
 
         request.setManualRedactions(new ManualRedactions());
@@ -409,11 +413,14 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                                                         .annotationId(annotationId)
                                                         .requestDate(OffsetDateTime.now())
                                                         .type("manual")
+                                                        .user("user")
                                                         .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
                                                         .positions(List.of(//
                                                                            new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
                                                                            new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
                                                                            new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
+                                                        .user("user")
+                                                        .fileId(request.getFileId())
                                                         .build()));
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId(annotationId)
@@ -421,6 +428,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .value("Expand to Hint")
                 .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2)))
                 .updateDictionary(false)
+                .user("user")
+                .fileId(request.getFileId())
                 .build();
         manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
         request.setManualRedactions(manualRedactions);
@@ -464,6 +473,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .value("Image")
                 .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
                 .updateDictionary(true)
+                .fileId(request.getFileId())
+                .user("user")
                 .build();
         manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
         request.setManualRedactions(manualRedactions);

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -1242,6 +1242,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -1895,6 +1895,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -1309,6 +1309,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl

@@ -801,6 +801,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -1330,6 +1330,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -253,6 +253,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -379,6 +379,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -279,6 +279,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -628,6 +628,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, active())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !resized(), active())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl

@@ -1890,6 +1890,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -1454,6 +1454,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
         $larger: TextEntity($type: type(), $entityType: entityType, !removed())
         $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
     end