diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 97a76599..af47a272 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -23,6 +23,7 @@ import java.time.ZoneOffset; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -2352,6 +2353,20 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { assertEquals(entityLog.getEntityLogEntry().size(), 3); } + @Test + public void testPurityRule() { + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + + AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entriesCount = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().toLowerCase(Locale.ENGLISH).startsWith("purity")).collect(Collectors.toList()).size(); + assertEquals(7, entriesCount); + } + private IdRemoval getIdRemoval(String id) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java index 2a66bcbb..435c0c50 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java @@ -109,4 +109,41 @@ public class RegExPatternTest { } } + @Test + public void testPurity() { + String text = "purity: 100% -> ok\n" + + "purity: <100% -> ok\n" + + "purity: 9% -> ok\n" + + "purity: <200% -> not ok\n" + + "purity 45%aa -> not ok\n" + + "purity: <45% -> ok\n" + + "purity: >45% -> ok\n" + + "purity: 101% -> not ok\n" + + "purity: 99.9% -> ok\n" + + "purity: 99,9% -> ok\n" + + "purity: 99,90% -> ok\n" + + "purity: aa 45% -> not ok\n" + + "purity: 99% -> ok\n" + + "purity: 99.99% -> ok\n" + + "purity: 100.00% -> ok?\n" + + "purity: <=45% -> not ok\n" + + "purity: >=45% -> not ok\n" + + "purity: <>45% -> not ok\n" + + "purity: =<45% -> not ok\n" + + "purity: =>45% -> not ok\n" + + "purity: aa45% -> not ok\n" + + "purity: 045% -> not ok\n" + + "purity: .45% -> not ok \n" + + "purity: 1000% -> not ok"; + String text2 = "Rule 39: Purity Hint Add Purity as Hint when Percent-Numbers is there Test Item: Soda Purity: 45% ← should be Hint Purity: <45% ← should be Hint Purity: >45% ← should be Hint Purity: 101% ← should ne be Hint because >100 % is not possible Purity: =>45% ← should be not Hint because additional symbols Purity: =<45% ← should be not Hint because additional symbols Purity: aa 45% ← should be not Hint because additional symbols Purity: 45% aa ← should be not Hint because additional symbols Purity: aa45% ← should be not Hint because additional symbols Purity: 45%aa ← should be not Hint because additional symbols Product-Code: EAK-L443 purity: 99% ← not Hint because case sensitive purity: >99% ← not Hint because case sensitive purity: <99% ← not Hint because case sensitive Supplier: GreenForce "; + Pattern p = Pattern.compile("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", Pattern.CASE_INSENSITIVE); + Matcher matcher = p.matcher(text); + while (matcher.find()) { + String match = matcher.group(0); + String match1 = matcher.group(1); + System.out.println("Group 0: " + match); + System.out.println("Group 1: " + match1); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 328a01fb..570c5ec8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -909,7 +909,7 @@ rule "ETC.0.0: Purity Hint" when $section: Section(containsStringIgnoreCase("purity")) then - entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section) + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", "hint_only", EntityType.HINT, 1, $section) .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 06aeeb78..80a4e78f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -1497,7 +1497,7 @@ rule "ETC.0.0: Purity Hint" when $section: Section(containsStringIgnoreCase("purity")) then - entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section) + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", "hint_only", EntityType.HINT, 1, $section) .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index 796d0a07..175b0355 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -646,7 +646,7 @@ rule "ETC.0.0: Purity Hint" when $section: Section(containsStringIgnoreCase("purity")) then - entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section) + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", "hint_only", EntityType.HINT, 1, $section) .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index a8d7470b..96eaf8e6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -627,7 +627,7 @@ rule "ETC.0.0: Purity Hint" when $section: Section(containsStringIgnoreCase("purity")) then - entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section) + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", "hint_only", EntityType.HINT, 1, $section) .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 1fc73ad7..648fdf20 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -1514,7 +1514,7 @@ rule "ETC.0.0: Purity Hint" when $section: Section(containsStringIgnoreCase("purity")) then - entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section) + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", "hint_only", EntityType.HINT, 1, $section) .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end diff --git a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt index da6022cd..de99d205 100644 --- a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt +++ b/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt @@ -1 +1 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;\nimport com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualRedactionApplicationService manualRedactionApplicationService\nglobal NerEntitiesAdapter nerEntitiesAdapter\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//------------------------------------ Syngenta specific rules ------------------------------------\n\n// Rule unit: SYN.1\nrule \"SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n $section: Section(containsString(\"CT\") || containsString(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n entityCreationService.byRegexIgnoreCase(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.skip(\"SYN.1.0\", \"\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n\n\n//------------------------------------ CBI rules ------------------------------------\n\n// Rule unit: CBI.0\nrule \"CBI.0.0: Redact CBI Authors (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.0\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"CBI.0.1: Redact CBI Authors (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.1\nrule \"CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.skip(\"CBI.1.0\", \"Address found for Non Vertebrate Study\");\n end\n\nrule \"CBI.1.1: Redact CBI Address (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.apply(\"CBI.1.1\", \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.2\nrule \"CBI.2.0: Don't redact genitive CBI_author\"\n when\n $entity: RedactionEntity(type == \"CBI_author\", anyMatch(textAfter, \"['’’'ʼˈ´`‘′ʻ’']s\"), isApplied())\n then\n RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), \"CBI_author\", EntityType.FALSE_POSITIVE, document);\n falsePositive.skip(\"CBI.2.0\", \"Genitive Author found\");\n insert(falsePositive);\n end\n\n\n// Rule unit: CBI.7\nrule \"CBI.7.0: Do not redact Names and Addresses if published information found in section without tables\"\n when\n $section: Section(!hasTables(),\n hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $section.getEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.0\",\n \"Published Information found in section\",\n $section.getEntitiesOfType(\"published_information\")\n );\n });\n end\n\nrule \"CBI.7.1: Do not redact Names and Addresses if published information found in same table row\"\n when\n $table: Table(hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.1\",\n \"Published Information found in row\",\n $table.getEntitiesOfTypeInSameRow(\"published_information\", redactionEntity)\n );\n });\n end\n\n\n// Rule unit: CBI.9\nrule \"CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.0\", \"Author(s) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.1\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.10\nrule \"CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.0\", \"Author(s) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.11\nrule \"CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -1\n when\n $table: Table(hasHeader(\"Author(s)\") && hasHeader(\"Vertebrate Study Y/N\"))\n then\n $table.getEntitiesOfType(\"CBI_author\").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));\n end\n\n\n// Rule unit: CBI.16\nrule \"CBI.16.0: Add CBI_author with \\\"et al.\\\" Regex (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.0\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n insert(entity);\n });\n end\n\nrule \"CBI.16.1: Add CBI_author with \\\"et al.\\\" Regex (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.1\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n });\n end\n\n\n// Rule unit: CBI.17\nrule \"CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species\") && containsString(\"Source\") && !containsString(\"Species:\") && !containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.0\", \"Line after \\\"Source\\\" in Test Organism Section\");\n insert(entity);\n });\n end\n\nrule \"CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species:\"), containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source:\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.1\", \"Line after \\\"Source:\\\" in Test Animals Section\");\n insert(entity);\n });\n end\n\n\n// Rule unit: CBI.20\nrule \"CBI.20.0: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.skip(\"CBI.20.0\", \"PERFORMING LABORATORY was found for non vertebrate study\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\nrule \"CBI.20.1: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.apply(\"CBI.20.1\", \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\n\n//------------------------------------ PII rules ------------------------------------\n\n// Rule unit: PII.0\nrule \"PII.0.0: Redact all PII (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.0\", \"Personal Information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"PII.0.1: Redact all PII (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.1\", \"Personal Information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: PII.1\nrule \"PII.1.0: Redact Emails by RegEx (Non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.0\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\nrule \"PII.1.1: Redact Emails by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.1\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\n\n// Rule unit: PII.2\nrule \"PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.0\", \"Found by Phone and Fax Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\nrule \"PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.1\", \"Found by Phone and Fax Regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\n\n// Rule unit: PII.9\nrule \"PII.9.0: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.0\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.1: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.1\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.2: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.2\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.3: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.3\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\n\n//------------------------------------ Other rules ------------------------------------\n\n// Rule unit: ETC.0\nrule \"ETC.0.0: Purity Hint\"\n when\n $section: Section(containsStringIgnoreCase(\"purity\"))\n then\n entityCreationService.byRegexIgnoreCase(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", \"hint_only\", EntityType.ENTITY, 1, $section)\n .forEach(hint -> {\n hint.addEngine(Engine.RULE);\n hint.skip(\"ETC.0.0\", \"\");\n });\n end\n\n\n// Rule unit: ETC.2\nrule \"ETC.2.0: Redact signatures (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.2.0: Redact signatures (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.3\nrule \"ETC.3.0: Redact logos (vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.0\", \"Logo Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.3.1: Redact logos (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.1\", \"Logo Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.5\nrule \"ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n not FileAttribute(label == \"Confidentiality\", value == \"confidential\")\n $dossierRedaction: RedactionEntity(type == \"dossier_redaction\")\n then\n $dossierRedaction.removeFromGraph();\n retract($dossierRedaction);\n end\n\n\n//------------------------------------ AI rules ------------------------------------\n\n// Rule unit: AI.0\nrule \"AI.0.0: add all NER Entities of type CBI_author\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"CBI_author\"))\n then\n nerEntities.streamEntitiesOfType(\"CBI_author\")\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))\n .forEach(entity -> insert(entity));\n end\n\n\n// Rule unit: AI.1\nrule \"AI.1.0: combine and add NER Entities as CBI_address\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"ORG\") || hasEntitiesOfType(\"STREET\") || hasEntitiesOfType(\"CITY\"))\n then\n nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)\n .map(boundary -> entityCreationService.byBoundary(boundary, \"CBI_address\", EntityType.RECOMMENDATION, document))\n .forEach(entity -> {\n entity.addEngine(Engine.NER);\n insert(entity);\n });\n end\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId)\n $entityToBeResized: RedactionEntity(matchesAnnotationId($id))\n then\n manualRedactionApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.setIgnored(true);\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.setIgnored(true);\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)\n $entityToForce: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToForce.apply(\"MAN.2.0\", \"Forced redaction\", $legalBasis);\n $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply image recategorization\"\n salience 128\n when\n ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)\n $image: Image($id == id)\n then\n $image.setImageType(ImageType.fromString($imageType));\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: RedactionEntity($type: type, $entityType: entityType)\n $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $contained.removeFromGraph();\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)\n $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $first.removeFromGraph();\n $second.removeFromGraph();\n RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);\n retract($first);\n retract($second);\n insert(mergedEntity);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)\n $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.removeFromGraph();\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)\n $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: RedactionEntity(entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $lowerRank.removeFromGraph();\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n" \ No newline at end of file +"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;\nimport com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualRedactionApplicationService manualRedactionApplicationService\nglobal NerEntitiesAdapter nerEntitiesAdapter\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//------------------------------------ Syngenta specific rules ------------------------------------\n\n// Rule unit: SYN.1\nrule \"SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n $section: Section(containsString(\"CT\") || containsString(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n entityCreationService.byRegexIgnoreCase(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.skip(\"SYN.1.0\", \"\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n\n\n//------------------------------------ CBI rules ------------------------------------\n\n// Rule unit: CBI.0\nrule \"CBI.0.0: Redact CBI Authors (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.0\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"CBI.0.1: Redact CBI Authors (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.1\nrule \"CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.skip(\"CBI.1.0\", \"Address found for Non Vertebrate Study\");\n end\n\nrule \"CBI.1.1: Redact CBI Address (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.apply(\"CBI.1.1\", \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.2\nrule \"CBI.2.0: Don't redact genitive CBI_author\"\n when\n $entity: RedactionEntity(type == \"CBI_author\", anyMatch(textAfter, \"['’’'ʼˈ´`‘′ʻ’']s\"), isApplied())\n then\n RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), \"CBI_author\", EntityType.FALSE_POSITIVE, document);\n falsePositive.skip(\"CBI.2.0\", \"Genitive Author found\");\n insert(falsePositive);\n end\n\n\n// Rule unit: CBI.7\nrule \"CBI.7.0: Do not redact Names and Addresses if published information found in section without tables\"\n when\n $section: Section(!hasTables(),\n hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $section.getEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.0\",\n \"Published Information found in section\",\n $section.getEntitiesOfType(\"published_information\")\n );\n });\n end\n\nrule \"CBI.7.1: Do not redact Names and Addresses if published information found in same table row\"\n when\n $table: Table(hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.1\",\n \"Published Information found in row\",\n $table.getEntitiesOfTypeInSameRow(\"published_information\", redactionEntity)\n );\n });\n end\n\n\n// Rule unit: CBI.9\nrule \"CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.0\", \"Author(s) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.1\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.10\nrule \"CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.0\", \"Author(s) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.11\nrule \"CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -1\n when\n $table: Table(hasHeader(\"Author(s)\") && hasHeader(\"Vertebrate Study Y/N\"))\n then\n $table.getEntitiesOfType(\"CBI_author\").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));\n end\n\n\n// Rule unit: CBI.16\nrule \"CBI.16.0: Add CBI_author with \\\"et al.\\\" Regex (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.0\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n insert(entity);\n });\n end\n\nrule \"CBI.16.1: Add CBI_author with \\\"et al.\\\" Regex (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.1\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n });\n end\n\n\n// Rule unit: CBI.17\nrule \"CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species\") && containsString(\"Source\") && !containsString(\"Species:\") && !containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.0\", \"Line after \\\"Source\\\" in Test Organism Section\");\n insert(entity);\n });\n end\n\nrule \"CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species:\"), containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source:\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.1\", \"Line after \\\"Source:\\\" in Test Animals Section\");\n insert(entity);\n });\n end\n\n\n// Rule unit: CBI.20\nrule \"CBI.20.0: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.skip(\"CBI.20.0\", \"PERFORMING LABORATORY was found for non vertebrate study\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\nrule \"CBI.20.1: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.apply(\"CBI.20.1\", \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\n\n//------------------------------------ PII rules ------------------------------------\n\n// Rule unit: PII.0\nrule \"PII.0.0: Redact all PII (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.0\", \"Personal Information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"PII.0.1: Redact all PII (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.1\", \"Personal Information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: PII.1\nrule \"PII.1.0: Redact Emails by RegEx (Non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.0\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\nrule \"PII.1.1: Redact Emails by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.1\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\n\n// Rule unit: PII.2\nrule \"PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.0\", \"Found by Phone and Fax Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\nrule \"PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.1\", \"Found by Phone and Fax Regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\n\n// Rule unit: PII.9\nrule \"PII.9.0: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.0\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.1: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.1\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.2: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.2\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.3: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.3\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\n\n//------------------------------------ Other rules ------------------------------------\n\n// Rule unit: ETC.0\nrule \"ETC.0.0: Purity Hint\"\n when\n $section: Section(containsStringIgnoreCase(\"purity\"))\n then\n entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", "hint_only", EntityType.HINT, 1, $section)\n .forEach(hint -> {\n hint.addEngine(Engine.RULE);\n hint.skip(\"ETC.0.0\", \"\");\n });\n end\n\n\n// Rule unit: ETC.2\nrule \"ETC.2.0: Redact signatures (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.2.0: Redact signatures (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.3\nrule \"ETC.3.0: Redact logos (vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.0\", \"Logo Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.3.1: Redact logos (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.1\", \"Logo Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.5\nrule \"ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n not FileAttribute(label == \"Confidentiality\", value == \"confidential\")\n $dossierRedaction: RedactionEntity(type == \"dossier_redaction\")\n then\n $dossierRedaction.removeFromGraph();\n retract($dossierRedaction);\n end\n\n\n//------------------------------------ AI rules ------------------------------------\n\n// Rule unit: AI.0\nrule \"AI.0.0: add all NER Entities of type CBI_author\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"CBI_author\"))\n then\n nerEntities.streamEntitiesOfType(\"CBI_author\")\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))\n .forEach(entity -> insert(entity));\n end\n\n\n// Rule unit: AI.1\nrule \"AI.1.0: combine and add NER Entities as CBI_address\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"ORG\") || hasEntitiesOfType(\"STREET\") || hasEntitiesOfType(\"CITY\"))\n then\n nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)\n .map(boundary -> entityCreationService.byBoundary(boundary, \"CBI_address\", EntityType.RECOMMENDATION, document))\n .forEach(entity -> {\n entity.addEngine(Engine.NER);\n insert(entity);\n });\n end\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId)\n $entityToBeResized: RedactionEntity(matchesAnnotationId($id))\n then\n manualRedactionApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.setIgnored(true);\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.setIgnored(true);\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)\n $entityToForce: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToForce.apply(\"MAN.2.0\", \"Forced redaction\", $legalBasis);\n $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply image recategorization\"\n salience 128\n when\n ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)\n $image: Image($id == id)\n then\n $image.setImageType(ImageType.fromString($imageType));\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: RedactionEntity($type: type, $entityType: entityType)\n $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $contained.removeFromGraph();\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)\n $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $first.removeFromGraph();\n $second.removeFromGraph();\n RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);\n retract($first);\n retract($second);\n insert(mergedEntity);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)\n $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.removeFromGraph();\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)\n $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: RedactionEntity(entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $lowerRank.removeFromGraph();\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n" \ No newline at end of file