From 0224e084248e95dab094b4c37cdb6a7af6b25c16 Mon Sep 17 00:00:00 2001 From: Corina Olariu Date: Tue, 4 Jun 2024 17:13:34 +0200 Subject: [PATCH] RED-9206 - Sections are no longer correctly separated from each other in the test file --- .../resources/drools/all_rules_documine.drl | 1 + .../v1/server/RedactionAcceptanceTest.java | 56 ++++++++++++++++++- .../ManualChangesEnd2EndTest.java | 17 +++++- .../resources/drools/acceptance_rules.drl | 1 + .../drools/all_redact_manager_rules.drl | 1 + .../test/resources/drools/documine_flora.drl | 1 + .../resources/drools/efsa_sanitisation.drl | 1 + .../src/test/resources/drools/rules.drl | 1 + .../src/test/resources/drools/rules_v2.drl | 1 + .../src/test/resources/drools/table_demo.drl | 1 + .../src/test/resources/drools/test_rules.drl | 1 + .../EFSA_sanitisation_GFL_v1/rules.drl | 1 + .../resources/all_redact_manager_rules.drl | 1 + .../src/main/resources/all_rules_documine.drl | 1 + 14 files changed, 81 insertions(+), 4 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl index e7b6d7bd..beabfd0c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl @@ -1454,6 +1454,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 3184d24c..fb314ba9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -85,7 +85,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); - when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) .type(DOSSIER_REDACTIONS_INDICATOR) @@ -170,6 +170,60 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { } + @Test + public void testLocalPublishedRemovalWithEntityContainedByEntityOfSameType() throws IOException { + + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf"); + System.out.println("Start Full integration test"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + System.out.println("Finished structure analysis"); + AnalyzeResult result = analyzeService.analyze(request); + System.out.println("Finished analysis"); + dictionary.get(PUBLISHED_INFORMATION_INDICATOR).add("Press"); + reanlysisVersions.put("Press", 2L); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L); + mockDictionaryCalls(0L); + + analyzeService.reanalyze(request); + + EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst() + .orElseThrow(); + assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:")); + var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst() + .orElseThrow(); + assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:")); + assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); + + var idRemoval = buildIdRemoval(publishedInformationEntry1.getId()); + + var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build(); + request.setManualRedactions(manualRedactions); + analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst() + .orElseThrow(); + var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst() + .orElseThrow(); + + assertEquals(EntryState.APPLIED, asyaLyon2.getState()); + + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); + + String outputFileName = OsUtils.getTemporaryDirectory() + "/AcceptanceTest.pdf"; + + try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { + fileOutputStream.write(annotateResponse.getDocument()); + } + + } + + private Stream findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, List sectionNumber) { return redactionLog.getEntityLogEntry() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 1c9bb5d1..e5bc9af8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -188,6 +188,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .addToAllDossiers(false) .updateDictionary(false) .requestDate(OffsetDateTime.now()) + .user("user") .build()); request.setManualRedactions(manualRedactions); analyzeService.reanalyze(request); @@ -233,17 +234,19 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { System.out.println("testManualRedaction"); long start = System.currentTimeMillis(); - String pdfFile = "files/syngenta/CustomeFiles/SinglePages/Single Table - Page87_S-Metolachlor_RAR_07_Volume_3CA_B-5_2018-09-06.pdf"; + String pdfFile = "files/syngenta/CustomerFiles/SinglePages/Single Table - Page87_S-Metolachlor_RAR_07_Volume_3CA_B-5_2018-09-06.pdf"; ManualRedactions manualRedactions = new ManualRedactions(); String manualAddId = UUID.randomUUID().toString(); - manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build())); + manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").user("user").requestDate(OffsetDateTime.now()).build())); manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder() .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") .legalBasis("Something") + .user("user") + .requestDate(OffsetDateTime.now()) .build())); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); @@ -262,7 +265,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { AnalyzeResult result = analyzeService.analyze(request); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); - manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build())); + manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").user("user").build())); manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder() .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") @@ -361,6 +364,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .type("vertebrate") .annotationId(oxfordUniversityPress.getId()) .fileId(TEST_FILE_ID) + .user("user") .build(); request.setManualRedactions(new ManualRedactions()); @@ -409,11 +413,14 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .annotationId(annotationId) .requestDate(OffsetDateTime.now()) .type("manual") + .user("user") .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated") .positions(List.of(// new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), // new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), // new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) // + .user("user") + .fileId(request.getFileId()) .build())); ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() .annotationId(annotationId) @@ -421,6 +428,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .value("Expand to Hint") .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2))) .updateDictionary(false) + .user("user") + .fileId(request.getFileId()) .build(); manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction)); request.setManualRedactions(manualRedactions); @@ -464,6 +473,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .value("Image") .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1))) .updateDictionary(true) + .fileId(request.getFileId()) + .user("user") .build(); manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction)); request.setManualRedactions(manualRedactions); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 91769e29..76d3ec84 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -1242,6 +1242,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 60ee405c..9b380905 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -1895,6 +1895,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 1f955cd5..77214ed4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1309,6 +1309,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index 0151d8f2..5c1ce6a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -801,6 +801,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index a9f94395..7720cf66 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1330,6 +1330,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index c7f23bb0..ed352cc0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -253,6 +253,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 73a7ff01..235469eb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -379,6 +379,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index f2856b72..6dc5c515 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -279,6 +279,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index 16b423ce..d1c66b0f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -628,6 +628,7 @@ rule "X.0.0: remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, active()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !resized(), active()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 9ea778d6..0239bdb3 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -1890,6 +1890,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index e7b6d7bd..beabfd0c 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -1454,6 +1454,7 @@ rule "X.0.0: Remove Entity contained by Entity of same type" $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then + $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end