From c71241288cf89d69746900c1d0c0c9274ef96d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Mon, 22 Jan 2024 13:38:47 +0100 Subject: [PATCH] RED-8305: Apply recategorizations by default, increase memory for unittests --- .../build.gradle.kts | 4 + .../v1/server/RedactionIntegrationTest.java | 50 ++++ .../resources/drools/acceptance_rules.drl | 268 +++++++++++++++++- .../drools/all_redact_manager_rules.drl | 20 +- .../test/resources/drools/documine_flora.drl | 10 +- .../drools/manual_redaction_rules.drl | 16 +- .../src/test/resources/drools/rules.drl | 19 +- .../src/test/resources/drools/rules_v2.drl | 20 +- .../src/test/resources/drools/table_demo.drl | 20 +- .../src/test/resources/drools/test_rules.drl | 20 +- .../resources/all_redact_manager_rules.drl | 18 +- .../src/main/resources/all_rules_documine.drl | 10 +- .../resources/default_rule_identifiers.txt | 9 +- 13 files changed, 434 insertions(+), 50 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index b6ed7ddc..4dd49196 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -77,9 +77,13 @@ tasks.test { configure { excludes = listOf("org/drools/**/*") } + minHeapSize = "512m" + maxHeapSize = "1024m" } + + tasks.named("bootBuildImage") { environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ") diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 46c47bf7..04df0c5e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1223,6 +1223,56 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { } + @Test + public void entityIsAppliedAfterRecateorize() throws IOException { + + AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf"); + ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json"); + + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), + imageServiceResponseFileResource.getInputStream()); + + System.out.println("Start Full integration test"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + System.out.println("Finished structure analysis"); + analyzeService.analyze(request); + System.out.println("Finished analysis"); + + request.setManualRedactions(ManualRedactions.builder() + .legalBasisChanges(Set.of(ManualLegalBasisChange.builder() + .annotationId("3029651d0842a625f2d23f8375c23600") + .section("[19, 2]: Paragraph: Contact point: LexCo Contact:") + .value("0049 331 441 551 14") + .requestDate(OffsetDateTime.now()) + .status(AnnotationStatus.APPROVED) + .fileId(TEST_FILE_ID) + .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") + .build())) + .recategorizations(Set.of(ManualRecategorization.builder() + .annotationId("3029651d0842a625f2d23f8375c23600") + .type("CBI_author") + .requestDate(OffsetDateTime.now()) + .status(AnnotationStatus.APPROVED) + .fileId(TEST_FILE_ID) + .build())) + .build()); + + analyzeService.reanalyze(request); + System.out.println("Finished reanalysis"); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var changedAnnotation = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600")) + .findFirst() + .get(); + + assertThat(changedAnnotation.getState() == EntryState.APPLIED).isTrue(); + + } + + @Test @SneakyThrows public void testRemovingAndAddingRedactionOnTheSameValue() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index ec5741cf..4a92feac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -374,6 +374,214 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" end +// Rule unit: PII.3 +rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(matchesRegex("[+]\\d{1,}")) + then + entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) + .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + +rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(matchesRegex("[+]\\d{1,}")) + then + entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + + +// Rule unit: PII.4 +rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No.", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.4.1: Redact line after contact information keywords (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No.", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + + +// Rule unit: PII.5 +rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + +rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + + +// Rule unit: PII.6 +rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + ) + .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + +rule "PII.6.1: Redact line between contact keywords (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + ) + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + + +// Rule unit: PII.7 +rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(getHeadline().containsString("applicant") || + getHeadline().containsString("Primary contact") || + getHeadline().containsString("Alternative contact") || + containsString("Applicant") || + containsString("Telephone number:")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + +rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(getHeadline().containsString("applicant") || + getHeadline().containsString("Primary contact") || + getHeadline().containsString("Alternative contact") || + containsString("Applicant") || + containsString("Telephone number:")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + + +// Rule unit: PII.8 +rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(containsStringIgnoreCase("producer of the plant protection") || + containsStringIgnoreCase("producer of the active substance") || + containsStringIgnoreCase("manufacturer of the active substance") || + containsStringIgnoreCase("manufacturer:") || + containsStringIgnoreCase("Producer or producers of the active substance")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.8.1: Redact contact information if producer is found (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(containsStringIgnoreCase("producer of the plant protection") || + containsStringIgnoreCase("producer of the active substance") || + containsStringIgnoreCase("manufacturer of the active substance") || + containsStringIgnoreCase("manufacturer:") || + containsStringIgnoreCase("Producer or producers of the active substance")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + + // Rule unit: PII.9 rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when @@ -394,6 +602,46 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ver end +// Rule unit: PII.10 +rule "PII.10.0: Redact study director abbreviation" + when + $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) + then + entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) + .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + + +// Rule unit: PII.11 +rule "PII.11.0: Redact On behalf of Sequani Ltd.:" + when + $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) + then + entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + + +// Rule unit: PII.12 +rule "PII.12.0: Expand PII entities with salutation prefix" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + then + entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + +rule "PII.12.1: Expand PII entities with salutation prefix" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + then + entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + + //------------------------------------ Other rules ------------------------------------ // Rule unit: ETC.0 @@ -597,15 +845,25 @@ rule "MAN.3.2: Apply image recategorization" retract($recategorization); end +rule "MAN.3.3: Apply recategorization entities by default" + salience 128 + when + $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) + then + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); + end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -615,6 +873,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end @@ -713,8 +973,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index ce48f64a..e0b255c9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -1415,15 +1415,25 @@ rule "MAN.3.2: Apply image recategorization" retract($recategorization); end +rule "MAN.3.3: Apply recategorization entities by default" + salience 128 + when + $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) + then + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); + end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -1433,6 +1443,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end @@ -1531,8 +1543,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 6d898478..eae06bf8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1278,7 +1278,7 @@ rule "MAN.3.3: Apply recategorization entities by default" when $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) then - $entity.apply("MAN.3.3", "Recategorized entities are applied by default."); + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); end @@ -1286,10 +1286,12 @@ rule "MAN.3.3: Apply recategorization entities by default" rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -1299,6 +1301,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index c9d42001..a97e0e71 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -187,15 +187,25 @@ rule "MAN.3.2: Apply image recategorization" retract($recategorization); end +rule "MAN.3.3: Apply recategorization entities by default" + salience 128 + when + $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) + then + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); + end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -205,6 +215,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index d54a84e8..136d7b20 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1005,22 +1005,25 @@ rule "MAN.3.2: Apply image recategorization" retract($recategorization); end -rule "MAN.3.4: Apply recategorization with legalbasis change by default" +rule "MAN.3.3: Apply recategorization entities by default" salience 128 when - $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), getManualOverwrite().getLegalBasis().isPresent()) + $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) then - $entity.apply("MAN.3.4", "Recategorized entities with legalbasis are applied by default."); + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -1030,6 +1033,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end @@ -1128,8 +1133,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 30ecca35..e0a31a54 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -211,15 +211,25 @@ rule "MAN.3.1: Apply entity recategorization of same type" retract($recategorization); end +rule "MAN.3.3: Apply recategorization entities by default" + salience 128 + when + $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) + then + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); + end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -229,6 +239,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end @@ -327,8 +339,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index a2b61013..cbf1002e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -337,15 +337,25 @@ rule "MAN.3.2: Apply image recategorization" retract($recategorization); end +rule "MAN.3.3: Apply recategorization entities by default" + salience 128 + when + $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) + then + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); + end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -355,6 +365,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end @@ -437,8 +449,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index fa8885e7..349f0bc5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -237,15 +237,25 @@ rule "MAN.3.2: Apply image recategorization" retract($recategorization); end +rule "MAN.3.3: Apply recategorization entities by default" + salience 128 + when + $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) + then + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); + end + // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -255,6 +265,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end @@ -353,8 +365,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 0de33d0a..0c6799e9 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -1437,22 +1437,24 @@ rule "MAN.3.2: Apply image recategorization" end -rule "MAN.3.4: Apply recategorization with legalbasis change by default" +rule "MAN.3.3: Apply recategorization entities by default" salience 128 when - $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), getManualOverwrite().getLegalBasis().isPresent()) + $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) then - $entity.apply("MAN.3.4", "Recategorized entities with legalbasis are applied by default."); + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); end // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -1462,6 +1464,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end @@ -1561,8 +1565,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index 730bfed7..72a2dcd0 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -1425,17 +1425,19 @@ rule "MAN.3.3: Apply recategorization entities by default" when $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) then - $entity.apply("MAN.3.3", "Recategorized entities are applied by default."); + $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); end // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 when - $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $imageToBeRecategorized: Image($id == id) then - $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); + update($imageToBeRecategorized) + retract($legalBasisChange) end rule "MAN.4.1: Apply legal basis change" @@ -1445,6 +1447,8 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + update($entityToBeChanged) + retract($legalBasisChange) end diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt index 6401382e..730c694c 100644 --- a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt +++ b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt @@ -1,11 +1,4 @@ -MAN.0.* -MAN.1.* -MAN.2.* -MAN.3.0 -MAN.3.1 -MAN.3.2 -MAN.3.4 -MAN.4.* +MAN.*.* X.0.0 X.1.0 X.2.0 -- 2.47.2