diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl index e0cf99ae..214730fe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl @@ -1562,7 +1562,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" retract($other); end -// Rule unit: X.9.0 +// Rule unit: X.9 rule "X.9.0: Merge mostly contained signatures" when $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) @@ -1572,7 +1572,7 @@ rule "X.9.0: Merge mostly contained signatures" $signature.addEngine(LayoutEngine.AI); end -// Rule unit: X.10.0 +// Rule unit: X.10 rule "X.10.0: remove false positives of ai" when $anyImage: Image(engines contains LayoutEngine.ALGORITHM) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index d9f65c3f..cf89ac40 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -224,7 +224,19 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study // Rule unit: CBI.16 -rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" +rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) + .forEach(entity -> { + entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + dictionary.recommendEverywhere(entity); + }); + end + +rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -232,12 +244,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(entity); }); end -rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" +rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -245,7 +257,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(entity); }); end @@ -270,7 +282,19 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with // Rule unit: CBI.20 -rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" +rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + dictionary.recommendEverywhere(laboratoryEntity); + }); + end + +rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -278,12 +302,12 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); + laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study"); dictionary.recommendEverywhere(laboratoryEntity); }); end -rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" +rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -291,7 +315,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -300,45 +324,77 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC //------------------------------------ PII rules ------------------------------------ // Rule unit: PII.0 -rule "PII.0.0: Redact all PII (non vertebrate study)" +rule "PII.0.0: Redact all PII" + when + $pii: TextEntity(type() == "PII", dictionaryEntry) + then + $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "PII.0.1: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "PII.0.1: Redact all PII (vertebrate study)" +rule "PII.0.2: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: PII.1 -rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" +rule "PII.1.0: Redact Emails by RegEx" + when + $section: Section(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" +rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.2 -rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" +rule "PII.2.0: Redact Phone and Fax by RegEx" + when + $section: Section(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Ph.") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) + then + entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) + .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || @@ -353,10 +409,10 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" +rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || @@ -371,34 +427,41 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.3 -rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" +rule "PII.3.0: Redact telephone numbers by RegEx" + when + $section: Section(matchesRegex("[+]\\d{1,}")) + then + entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) + .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" +rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.4 -rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" +rule "PII.4.0: Redact line after contact information keywords" when - not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -424,9 +487,9 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end -rule "PII.4.1: Redact line after contact information keywords (vertebrate study)" +rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -452,9 +515,49 @@ rule "PII.4.1: Redact line after contact information keywords (vertebrate study) .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end +rule "PII.4.2: Redact line after contact information keywords (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No.", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + // Rule unit: PII.5 -rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)" +rule "PII.5.0: Redact line after contact information keywords reduced" + when + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.5.1: Redact line after contact information keywords reduced (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", @@ -464,10 +567,10 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)" +rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", @@ -477,12 +580,23 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.6 -rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" +rule "PII.6.0: Redact line between contact keywords" + when + $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + ) + .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) @@ -491,10 +605,10 @@ rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.6.1: Redact line between contact keywords (vertebrate study)" +rule "PII.6.2: Redact line between contact keywords (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) @@ -503,12 +617,29 @@ rule "PII.6.1: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.7 -rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" +rule "PII.7.0: Redact contact information if applicant is found" + when + $section: Section(getHeadline().containsString("applicant") || + getHeadline().containsString("Primary contact") || + getHeadline().containsString("Alternative contact") || + containsString("Applicant") || + containsString("Telephone number:")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || @@ -523,10 +654,10 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)" +rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || @@ -541,14 +672,13 @@ rule "PII.7.1: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.8 -rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)" +rule "PII.8.0: Redact contact information if producer is found" when - not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -564,7 +694,25 @@ rule "PII.8.0: Redact contact information if producer is found (non vertebrate s .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end -rule "PII.8.1: Redact contact information if producer is found (vertebrate study)" +rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(containsStringIgnoreCase("producer of the plant protection") || + containsStringIgnoreCase("producer of the active substance") || + containsStringIgnoreCase("manufacturer of the active substance") || + containsStringIgnoreCase("manufacturer:") || + containsStringIgnoreCase("Producer or producers of the active substance")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || @@ -579,27 +727,35 @@ rule "PII.8.1: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.9 -rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" +rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" + when + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" +rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -656,41 +812,56 @@ rule "ETC.0.0: Purity Hint" // Rule unit: ETC.2 -rule "ETC.2.0: Redact signatures (non vertebrate study)" +rule "ETC.2.0: Redact signatures" + when + $signature: Image(imageType == ImageType.SIGNATURE) + then + $signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "ETC.2.1: Redact signatures (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "ETC.2.1: Redact signatures (vertebrate study)" +rule "ETC.2.2: Redact signatures (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: ETC.3 -rule "ETC.3.0: Skip logos (non vertebrate study)" +rule "ETC.3.0: Redact logos" + when + $logo: Image(imageType == ImageType.LOGO) + then + $logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "ETC.3.1: Skip logos (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.skip("ETC.3.0", "Logo Found"); + $logo.skip("ETC.3.1", "Logo Found"); end -rule "ETC.3.1: Redact logos (vertebrate study)" +rule "ETC.3.2: Redact logos (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end + // Rule unit: ETC.5 rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'" when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 75dec75e..1e106aff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -159,17 +159,18 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate" rule "CBI.3.1: Redacted because table row contains a vertebrate" when - $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList() - $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.applyWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( "CBI.3.1", "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)", - $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress) + $table.getEntitiesOfTypeInSameRow("vertebrate", entity) ); + }); end rule "CBI.3.2: Do not redact because Section does not contain a vertebrate" @@ -208,21 +209,23 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo }); end -rule "CBI.4.1: Redacted because table row contains a vertebrate" +rule "CBI.4.1: Don't redact authors or addresses which appear in the same row as a vertebrate and a no_redaction_indicator" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList() - TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList() - $tableCell: TableCell(row == $row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("vertebrate"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.skipWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.skipWithReferences( "CBI.4.1", "Vertebrate but a no redaction indicator found", Stream.concat( - $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(), - $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList() + $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(), + $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList() ); + }); end @@ -249,20 +252,22 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList() - TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList() - $tableCell: TableCell(row == $row) from $table.streamTableCells().toList() - $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("redaction_indicator"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $entity.applyWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( "CBI.5.1", "no_redaction_indicator but also redaction_indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)", Stream.concat( - $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(), - $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList() + $table.getEntitiesOfTypeInSameRow("redaction_indicator", entity).stream(), + $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList() ); + }); end @@ -352,17 +357,18 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity" rule "CBI.8.1: Redacted because table row contains must_redact entity" when - $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList() - $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.applyWithReferences( - "CBI.8.1", - "Must_redact found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress) - ); + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( + "CBI.8.1", + "Must_redact found", + "Reg (EC) No 1107/2009 Art. 63 (2g)", + $table.getEntitiesOfTypeInSameRow("must_redact", entity) + ); + }); end @@ -434,7 +440,22 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study // Rule unit: CBI.12 -rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" +rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + $rowCell: TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() + then + entityCreationService.bySemanticNode($rowCell, "must_redact", EntityType.HINT) + .ifPresent(yesEntity -> { + yesEntity.skip("CBI.12.0", "must_redact"); + }); + end + +rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -446,13 +467,13 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end -rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" +rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -465,13 +486,13 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end -rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" +rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" when $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() @@ -480,7 +501,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList() then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) - .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study")); + .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study")); end @@ -498,9 +519,12 @@ rule "CBI.13.0: Ignore CBI Address recommendations" // Rule unit: CBI.14 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" when + $section: Section(containsStringIgnoreCase("batches produced at")) $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at")) then $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + entityCreationService.byString("batches produced at", "must_redact", EntityType.HINT, $section) + .forEach(entity -> entity.skip("CBI.14.0", "must_redact")); end @@ -521,10 +545,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) then - entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section) - .toList(); + entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $section) + .forEach(entity -> entity.skip("CBI.15.0", "must_redact")); - $section.getEntitiesOfType(List.of($keyword, $residueKeyword)) + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -542,8 +566,8 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio $residueKeyword: String() from List.of("determination of residues", "determination of total residues") $table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) then - entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table) - .toList(); + entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $table) + .forEach(entity -> entity.skip("CBI.15.1", "must_redact")); $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword)) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) @@ -552,7 +576,19 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio // Rule unit: CBI.16 -rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" +rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) + .forEach(entity -> { + entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + dictionary.recommendEverywhere(entity); + }); + end + +rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -560,12 +596,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(entity); }); end -rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" +rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -573,7 +609,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(entity); }); end @@ -631,7 +667,19 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" // Rule unit: CBI.20 -rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" +rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + dictionary.recommendEverywhere(laboratoryEntity); + }); + end + +rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -639,12 +687,12 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); + laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study"); dictionary.recommendEverywhere(laboratoryEntity); }); end -rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" +rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -652,12 +700,11 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(laboratoryEntity); }); end - // Rule unit: CBI.21 rule "CBI.21.0: Redact short Authors section (non vertebrate study)" when @@ -699,53 +746,92 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n //------------------------------------ PII rules ------------------------------------ // Rule unit: PII.0 -rule "PII.0.0: Redact all PII (non vertebrate study)" +rule "PII.0.0: Redact all PII" + when + $pii: TextEntity(type() == "PII", dictionaryEntry) + then + $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "PII.0.1: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "PII.0.1: Redact all PII (vertebrate study)" +rule "PII.0.2: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: PII.1 -rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" +rule "PII.1.0: Redact Emails by RegEx" + when + $section: Section(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" +rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end -rule "PII.1.2: Redact typoed Emails with indicator" +rule "PII.1.3: Redact typoed Emails with indicator" when $section: Section(containsString("@") || containsStringIgnoreCase("mail")) then entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end +rule "PII.1.4: Redact typoed Emails with indicator" + when + $section: Section(containsString("@") || containsStringIgnoreCase("mail")) + then + entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> emailEntity.redact("PII.1.4", "Personal information found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end // Rule unit: PII.2 -rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" +rule "PII.2.0: Redact Phone and Fax by RegEx" + when + $section: Section(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Ph.") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) + then + entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) + .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || @@ -760,10 +846,10 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" +rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || @@ -778,42 +864,48 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end -rule "PII.2.2: Redact phone numbers without indicators" +rule "PII.2.3: Redact phone numbers without indicators" when $section: Section(containsString("+")) then entityCreationService.byRegex("(\\+[\\dO]{1,2} )(\\([\\dO]{1,3}\\))?[\\d\\-O ]{8,15}", "PII", EntityType.ENTITY, $section) - .forEach(entity -> entity.redact("PII.2.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end // Rule unit: PII.3 -rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" +rule "PII.3.0: Redact telephone numbers by RegEx" + when + $section: Section(matchesRegex("[+]\\d{1,}")) + then + entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) + .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" +rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end - // Rule unit: PII.4 -rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" +rule "PII.4.0: Redact line after contact information keywords" when - not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -839,9 +931,9 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end -rule "PII.4.1: Redact line after contact information keywords (vertebrate study)" +rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -867,9 +959,49 @@ rule "PII.4.1: Redact line after contact information keywords (vertebrate study) .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end +rule "PII.4.2: Redact line after contact information keywords (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No.", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + // Rule unit: PII.5 -rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)" +rule "PII.5.0: Redact line after contact information keywords reduced" + when + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.5.1: Redact line after contact information keywords reduced (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", @@ -879,10 +1011,10 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)" +rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", @@ -892,12 +1024,23 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.6 -rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" +rule "PII.6.0: Redact line between contact keywords" + when + $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + ) + .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) @@ -906,10 +1049,10 @@ rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.6.1: Redact line between contact keywords (vertebrate study)" +rule "PII.6.2: Redact line between contact keywords (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) @@ -918,12 +1061,29 @@ rule "PII.6.1: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.7 -rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" +rule "PII.7.0: Redact contact information if applicant is found" + when + $section: Section(getHeadline().containsString("applicant") || + getHeadline().containsString("Primary contact") || + getHeadline().containsString("Alternative contact") || + containsString("Applicant") || + containsString("Telephone number:")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || @@ -938,10 +1098,10 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)" +rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || @@ -956,14 +1116,13 @@ rule "PII.7.1: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.8 -rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)" +rule "PII.8.0: Redact contact information if producer is found" when - not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -979,7 +1138,25 @@ rule "PII.8.0: Redact contact information if producer is found (non vertebrate s .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end -rule "PII.8.1: Redact contact information if producer is found (vertebrate study)" +rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(containsStringIgnoreCase("producer of the plant protection") || + containsStringIgnoreCase("producer of the active substance") || + containsStringIgnoreCase("manufacturer of the active substance") || + containsStringIgnoreCase("manufacturer:") || + containsStringIgnoreCase("Producer or producers of the active substance")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || @@ -994,27 +1171,35 @@ rule "PII.8.1: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.9 -rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" +rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" + when + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" +rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -1049,6 +1234,16 @@ rule "PII.12.0: Expand PII entities with salutation prefix" end +// Rule unit: PII.12 +rule "PII.12.1: Expand PII entities with salutation prefix" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + then + entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + // Rule unit: PII.13 rule "PII.13.0: Add recommendation for PII after Contact Person" when @@ -1091,38 +1286,52 @@ rule "ETC.1.0: Redact Purity" // Rule unit: ETC.2 -rule "ETC.2.0: Redact signatures (non vertebrate study)" +rule "ETC.2.0: Redact signatures" + when + $signature: Image(imageType == ImageType.SIGNATURE) + then + $signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "ETC.2.1: Redact signatures (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "ETC.2.1: Redact signatures (vertebrate study)" +rule "ETC.2.2: Redact signatures (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: ETC.3 -rule "ETC.3.0: Skip logos (non vertebrate study)" +rule "ETC.3.0: Redact logos" + when + $logo: Image(imageType == ImageType.LOGO) + then + $logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "ETC.3.1: Skip logos (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.skip("ETC.3.0", "Logo Found"); + $logo.skip("ETC.3.1", "Logo Found"); end -rule "ETC.3.1: Redact logos (vertebrate study)" +rule "ETC.3.2: Redact logos (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -1212,7 +1421,6 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end - // Rule unit: ETC.9 rule "ETC.9.0: Redact skipped impurities" when @@ -1230,7 +1438,6 @@ rule "ETC.9.1: Redact impurities" $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); end - // Rule unit: ETC.10 rule "ETC.10.0: Redact Product Composition Information" when @@ -1239,7 +1446,6 @@ rule "ETC.10.0: Redact Product Composition Information" $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009"); end - // Rule unit: ETC.11 rule "ETC.11.0: Recommend first line in table cell with name and address of owner" when @@ -1252,6 +1458,23 @@ rule "ETC.11.0: Recommend first line in table cell with name and address of owne end +// Rule unit: ETC.12 +rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $dossierRedaction: TextEntity(type() == "dossier_redaction") + then + $dossierRedaction.redact("ETC.12.0", "Dossier dictionary entry found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $dossierRedaction: TextEntity(type() == "dossier_redaction") + then + $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + //------------------------------------ AI rules ------------------------------------ // Rule unit: AI.0 @@ -1297,8 +1520,7 @@ rule "AI.3.0: Recommend authors from AI as PII" .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document)); end - -//------------------------------------ Manual changes rules ------------------------------------ +//------------------------------------ Manual redaction rules ------------------------------------ // Rule unit: MAN.0 rule "MAN.0.0: Apply manual resize redaction" @@ -1418,6 +1640,7 @@ rule "MAN.3.2: Apply image recategorization" retract($recategorization); end + rule "MAN.3.3: Apply recategorization entities by default" salience 128 when @@ -1426,7 +1649,6 @@ rule "MAN.3.3: Apply recategorization entities by default" $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); end - // Rule unit: MAN.4 rule "MAN.4.0: Apply legal basis change" salience 128 @@ -1587,7 +1809,6 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" retract($other); end - // Rule unit: X.9 rule "X.9.0: Merge mostly contained signatures" when @@ -1598,7 +1819,6 @@ rule "X.9.0: Merge mostly contained signatures" $signature.addEngine(LayoutEngine.AI); end - // Rule unit: X.10 rule "X.10.0: remove false positives of ai" when @@ -1608,7 +1828,6 @@ rule "X.10.0: remove false positives of ai" $aiSignature.remove("X.10.0", "Removed because false positive"); end - //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index f75d7e92..6505b0de 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -102,17 +102,18 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate" rule "CBI.3.1: Redacted because table row contains a vertebrate" when - $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList() - $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.applyWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( "CBI.3.1", "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)", - $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress) + $table.getEntitiesOfTypeInSameRow("vertebrate", entity) ); + }); end rule "CBI.3.2: Do not redact because Section does not contain a vertebrate" @@ -151,21 +152,23 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo }); end -rule "CBI.4.1: Redacted because table row contains a vertebrate" +rule "CBI.4.1: Don't redact authors or addresses which appear in the same row as a vertebrate and a no_redaction_indicator" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList() - TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList() - $tableCell: TableCell(row == $row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("vertebrate"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.skipWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.skipWithReferences( "CBI.4.1", "Vertebrate but a no redaction indicator found", Stream.concat( - $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(), - $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList() + $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(), + $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList() ); + }); end @@ -192,20 +195,22 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList() - TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList() - $tableCell: TableCell(row == $row) from $table.streamTableCells().toList() - $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("redaction_indicator"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $entity.applyWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( "CBI.5.1", "no_redaction_indicator but also redaction_indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)", Stream.concat( - $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(), - $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList() + $table.getEntitiesOfTypeInSameRow("redaction_indicator", entity).stream(), + $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList() ); + }); end @@ -227,17 +232,18 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity" rule "CBI.8.1: Redacted because table row contains must_redact entity" when - $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList() - $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.applyWithReferences( - "CBI.8.1", - "Must_redact found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress) - ); + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( + "CBI.8.1", + "Must_redact found", + "Reg (EC) No 1107/2009 Art. 63 (2g)", + $table.getEntitiesOfTypeInSameRow("must_redact", entity) + ); + }); end @@ -281,7 +287,22 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study // Rule unit: CBI.12 -rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" +rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + $rowCell: TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() + then + entityCreationService.bySemanticNode($rowCell, "must_redact", EntityType.HINT) + .ifPresent(yesEntity -> { + yesEntity.skip("CBI.12.0", "must_redact"); + }); + end + +rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -293,13 +314,13 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end -rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" +rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -312,13 +333,13 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end -rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" +rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" when $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() @@ -327,16 +348,19 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList() then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) - .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study")); + .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study")); end // Rule unit: CBI.14 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" when + $section: Section(containsStringIgnoreCase("batches produced at")) $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at")) then $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + entityCreationService.byString("batches produced at", "must_redact", EntityType.HINT, $section) + .forEach(entity -> entity.skip("CBI.14.0", "must_redact")); end @@ -357,10 +381,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) then - entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section) - .toList(); + entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $section) + .forEach(entity -> entity.skip("CBI.15.0", "must_redact")); - $section.getEntitiesOfType(List.of($keyword, $residueKeyword)) + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -378,8 +402,8 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio $residueKeyword: String() from List.of("determination of residues", "determination of total residues") $table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) then - entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table) - .toList(); + entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $table) + .forEach(entity -> entity.skip("CBI.15.1", "must_redact")); $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword)) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) @@ -388,7 +412,19 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio // Rule unit: CBI.16 -rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" +rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) + .forEach(entity -> { + entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + dictionary.recommendEverywhere(entity); + }); + end + +rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -396,12 +432,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(entity); }); end -rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" +rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -409,7 +445,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(entity); }); end @@ -467,7 +503,19 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" // Rule unit: CBI.20 -rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" +rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + dictionary.recommendEverywhere(laboratoryEntity); + }); + end + +rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -475,12 +523,12 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); + laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study"); dictionary.recommendEverywhere(laboratoryEntity); }); end -rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" +rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -488,7 +536,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -496,48 +544,63 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC //------------------------------------ PII rules ------------------------------------ + // Rule unit: PII.0 -rule "PII.0.0: Redact all PII (non vertebrate study)" +rule "PII.0.0: Redact all PII" + when + $pii: TextEntity(type() == "PII", dictionaryEntry) + then + $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "PII.0.1: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "PII.0.1: Redact all PII (vertebrate study)" +rule "PII.0.2: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: PII.1 -rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" +rule "PII.1.0: Redact Emails by RegEx" + when + $section: Section(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" +rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.4 -rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" +rule "PII.4.0: Redact line after contact information keywords" when - not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -563,9 +626,9 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end -rule "PII.4.1: Redact line after contact information keywords (vertebrate study)" +rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -591,9 +654,48 @@ rule "PII.4.1: Redact line after contact information keywords (vertebrate study) .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end +rule "PII.4.2: Redact line after contact information keywords (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No.", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + // Rule unit: PII.6 -rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" +rule "PII.6.0: Redact line between contact keywords" + when + $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + ) + .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) @@ -602,10 +704,10 @@ rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.6.1: Redact line between contact keywords (vertebrate study)" +rule "PII.6.2: Redact line between contact keywords (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) @@ -614,12 +716,29 @@ rule "PII.6.1: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.7 -rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" +rule "PII.7.0: Redact contact information if applicant is found" + when + $section: Section(getHeadline().containsString("applicant") || + getHeadline().containsString("Primary contact") || + getHeadline().containsString("Alternative contact") || + containsString("Applicant") || + containsString("Telephone number:")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || @@ -634,10 +753,10 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)" +rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || @@ -652,14 +771,13 @@ rule "PII.7.1: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.8 -rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)" +rule "PII.8.0: Redact contact information if producer is found" when - not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -675,7 +793,25 @@ rule "PII.8.0: Redact contact information if producer is found (non vertebrate s .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end -rule "PII.8.1: Redact contact information if producer is found (vertebrate study)" +rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(containsStringIgnoreCase("producer of the plant protection") || + containsStringIgnoreCase("producer of the active substance") || + containsStringIgnoreCase("manufacturer of the active substance") || + containsStringIgnoreCase("manufacturer:") || + containsStringIgnoreCase("Producer or producers of the active substance")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || @@ -690,27 +826,35 @@ rule "PII.8.1: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.9 -rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" +rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" + when + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" +rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -757,38 +901,52 @@ rule "ETC.1.0: Redact Purity" // Rule unit: ETC.2 -rule "ETC.2.0: Redact signatures (non vertebrate study)" +rule "ETC.2.0: Redact signatures" + when + $signature: Image(imageType == ImageType.SIGNATURE) + then + $signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "ETC.2.1: Redact signatures (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "ETC.2.1: Redact signatures (vertebrate study)" +rule "ETC.2.2: Redact signatures (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: ETC.3 -rule "ETC.3.0: Skip logos (non vertebrate study)" +rule "ETC.3.0: Redact logos" + when + $logo: Image(imageType == ImageType.LOGO) + then + $logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "ETC.3.1: Skip logos (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.skip("ETC.3.0", "Logo Found"); + $logo.skip("ETC.3.1", "Logo Found"); end -rule "ETC.3.1: Redact logos (vertebrate study)" +rule "ETC.3.2: Redact logos (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index eebf5995..6126142e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -82,12 +82,12 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" //------------------------------------ PII rules ------------------------------------ // Rule unit: PII.0 -rule "PII.0.0: Redact all PII (non vertebrate study)" +rule "PII.0.1: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index ecdd6e65..16b423ce 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -295,47 +295,47 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC //------------------------------------ PII rules ------------------------------------ // Rule unit: PII.0 -rule "PII.0.0: Redact all PII (non vertebrate study)" +rule "PII.0.1: Redact all PII (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "PII.0.1: Redact all PII (vertebrate study)" +rule "PII.0.2: Redact all PII (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: PII.1 -rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" +rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" +rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.2 -rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" +ule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -348,12 +348,12 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" +rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -366,45 +366,45 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.9 -rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" +rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + +rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + +rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)" +rule "PII.9.4: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)" when FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); - end - -rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" - when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) - then - entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); - end - -rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)" - when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) - then - entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.apply("PII.9.4", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -421,7 +421,7 @@ rule "ETC.0.0: Purity Hint" // Rule unit: ETC.2 -rule "ETC.2.0: Redact signatures (non vertebrate study)" +rule "ETC.2.1: Redact signatures (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value == "Yes") $signature: Image(imageType == ImageType.SIGNATURE) @@ -431,7 +431,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" // Rule unit: ETC.3 -rule "ETC.3.0: Redact logos (vertebrate study)" +rule "ETC.3.2: Redact logos (vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value == "Yes") $logo: Image(imageType == ImageType.LOGO) @@ -439,7 +439,7 @@ rule "ETC.3.0: Redact logos (vertebrate study)" $logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "ETC.3.1: Redact logos (non vertebrate study)" +rule "ETC.3.3: Redact logos (non vertebrate study)" when FileAttribute(label == "Vertebrate Study", value == "Yes") $logo: Image(imageType == ImageType.LOGO) diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 9d43763e..1e106aff 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -159,20 +159,20 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate" rule "CBI.3.1: Redacted because table row contains a vertebrate" when - $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList() - $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.applyWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( "CBI.3.1", "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)", - $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress) + $table.getEntitiesOfTypeInSameRow("vertebrate", entity) ); + }); end - rule "CBI.3.2: Do not redact because Section does not contain a vertebrate" when $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) @@ -209,21 +209,23 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo }); end -rule "CBI.4.1: Redacted because table row contains a vertebrate" +rule "CBI.4.1: Don't redact authors or addresses which appear in the same row as a vertebrate and a no_redaction_indicator" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList() - TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList() - $tableCell: TableCell(row == $row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("vertebrate"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.skipWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.skipWithReferences( "CBI.4.1", "Vertebrate but a no redaction indicator found", Stream.concat( - $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(), - $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList() + $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(), + $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList() ); + }); end @@ -250,20 +252,22 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList() - TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList() - $tableCell: TableCell(row == $row) from $table.streamTableCells().toList() - $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("redaction_indicator"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $entity.applyWithReferences( + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( "CBI.5.1", "no_redaction_indicator but also redaction_indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)", Stream.concat( - $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(), - $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList() + $table.getEntitiesOfTypeInSameRow("redaction_indicator", entity).stream(), + $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList() ); + }); end @@ -353,17 +357,18 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity" rule "CBI.8.1: Redacted because table row contains must_redact entity" when - $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList() - $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() + $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $authorOrAddress.applyWithReferences( - "CBI.8.1", - "Must_redact found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress) - ); + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact")) + .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) + .forEach(entity -> { + entity.applyWithReferences( + "CBI.8.1", + "Must_redact found", + "Reg (EC) No 1107/2009 Art. 63 (2g)", + $table.getEntitiesOfTypeInSameRow("must_redact", entity) + ); + }); end @@ -435,7 +440,22 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study // Rule unit: CBI.12 -rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" +rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + $rowCell: TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() + then + entityCreationService.bySemanticNode($rowCell, "must_redact", EntityType.HINT) + .ifPresent(yesEntity -> { + yesEntity.skip("CBI.12.0", "must_redact"); + }); + end + +rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -447,13 +467,13 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end -rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" +rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -466,13 +486,13 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end -rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" +rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" when $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() @@ -481,7 +501,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList() then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) - .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study")); + .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study")); end @@ -499,9 +519,12 @@ rule "CBI.13.0: Ignore CBI Address recommendations" // Rule unit: CBI.14 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" when + $section: Section(containsStringIgnoreCase("batches produced at")) $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at")) then $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + entityCreationService.byString("batches produced at", "must_redact", EntityType.HINT, $section) + .forEach(entity -> entity.skip("CBI.14.0", "must_redact")); end @@ -522,10 +545,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) then - entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section) - .toList(); + entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $section) + .forEach(entity -> entity.skip("CBI.15.0", "must_redact")); - $section.getEntitiesOfType(List.of($keyword, $residueKeyword)) + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -543,8 +566,8 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio $residueKeyword: String() from List.of("determination of residues", "determination of total residues") $table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) then - entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table) - .toList(); + entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $table) + .forEach(entity -> entity.skip("CBI.15.1", "must_redact")); $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword)) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) @@ -553,7 +576,19 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio // Rule unit: CBI.16 -rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" +rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) + .forEach(entity -> { + entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + dictionary.recommendEverywhere(entity); + }); + end + +rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -561,12 +596,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(entity); }); end -rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" +rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -574,7 +609,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(entity); }); end @@ -632,7 +667,19 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" // Rule unit: CBI.20 -rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" +rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + dictionary.recommendEverywhere(laboratoryEntity); + }); + end + +rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -640,12 +687,12 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); + laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study"); dictionary.recommendEverywhere(laboratoryEntity); }); end -rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" +rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -653,7 +700,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -699,53 +746,92 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n //------------------------------------ PII rules ------------------------------------ // Rule unit: PII.0 -rule "PII.0.0: Redact all PII (non vertebrate study)" +rule "PII.0.0: Redact all PII" + when + $pii: TextEntity(type() == "PII", dictionaryEntry) + then + $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "PII.0.1: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "PII.0.1: Redact all PII (vertebrate study)" +rule "PII.0.2: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: PII.1 -rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" +rule "PII.1.0: Redact Emails by RegEx" + when + $section: Section(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" +rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end -rule "PII.1.2: Redact typoed Emails with indicator" +rule "PII.1.3: Redact typoed Emails with indicator" when $section: Section(containsString("@") || containsStringIgnoreCase("mail")) then entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end +rule "PII.1.4: Redact typoed Emails with indicator" + when + $section: Section(containsString("@") || containsStringIgnoreCase("mail")) + then + entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> emailEntity.redact("PII.1.4", "Personal information found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end // Rule unit: PII.2 -rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" +rule "PII.2.0: Redact Phone and Fax by RegEx" + when + $section: Section(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Ph.") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) + then + entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) + .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || @@ -760,10 +846,10 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" +rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || @@ -778,41 +864,48 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end -rule "PII.2.2: Redact phone numbers without indicators" +rule "PII.2.3: Redact phone numbers without indicators" when $section: Section(containsString("+")) then entityCreationService.byRegex("(\\+[\\dO]{1,2} )(\\([\\dO]{1,3}\\))?[\\d\\-O ]{8,15}", "PII", EntityType.ENTITY, $section) - .forEach(entity -> entity.redact("PII.2.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end // Rule unit: PII.3 -rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" +rule "PII.3.0: Redact telephone numbers by RegEx" + when + $section: Section(matchesRegex("[+]\\d{1,}")) + then + entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) + .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" +rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.4 -rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" +rule "PII.4.0: Redact line after contact information keywords" when - not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -838,9 +931,9 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end -rule "PII.4.1: Redact line after contact information keywords (vertebrate study)" +rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -866,9 +959,49 @@ rule "PII.4.1: Redact line after contact information keywords (vertebrate study) .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end +rule "PII.4.2: Redact line after contact information keywords (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No.", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + // Rule unit: PII.5 -rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)" +rule "PII.5.0: Redact line after contact information keywords reduced" + when + $contactKeyword: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:") + $section: Section(containsString($contactKeyword)) + then + entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) + .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.5.1: Redact line after contact information keywords reduced (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", @@ -878,10 +1011,10 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)" +rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", @@ -891,12 +1024,23 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.6 -rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" +rule "PII.6.0: Redact line between contact keywords" + when + $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + ) + .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) @@ -905,10 +1049,10 @@ rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.6.1: Redact line between contact keywords (vertebrate study)" +rule "PII.6.2: Redact line between contact keywords (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) @@ -917,12 +1061,29 @@ rule "PII.6.1: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.7 -rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" +rule "PII.7.0: Redact contact information if applicant is found" + when + $section: Section(getHeadline().containsString("applicant") || + getHeadline().containsString("Primary contact") || + getHeadline().containsString("Alternative contact") || + containsString("Applicant") || + containsString("Telephone number:")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || @@ -937,10 +1098,10 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)" +rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || @@ -955,14 +1116,13 @@ rule "PII.7.1: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.8 -rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)" +rule "PII.8.0: Redact contact information if producer is found" when - not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -978,7 +1138,25 @@ rule "PII.8.0: Redact contact information if producer is found (non vertebrate s .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end -rule "PII.8.1: Redact contact information if producer is found (vertebrate study)" +rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $section: Section(containsStringIgnoreCase("producer of the plant protection") || + containsStringIgnoreCase("producer of the active substance") || + containsStringIgnoreCase("manufacturer of the active substance") || + containsStringIgnoreCase("manufacturer:") || + containsStringIgnoreCase("Producer or producers of the active substance")) + then + Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", + "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), + Stream.concat( + entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), + entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) + )) + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || @@ -993,27 +1171,35 @@ rule "PII.8.1: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end // Rule unit: PII.9 -rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" +rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" + when + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); + end + +rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" +rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -1100,38 +1286,52 @@ rule "ETC.1.0: Redact Purity" // Rule unit: ETC.2 -rule "ETC.2.0: Redact signatures (non vertebrate study)" +rule "ETC.2.0: Redact signatures" + when + $signature: Image(imageType == ImageType.SIGNATURE) + then + $signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "ETC.2.1: Redact signatures (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "ETC.2.1: Redact signatures (vertebrate study)" +rule "ETC.2.2: Redact signatures (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end // Rule unit: ETC.3 -rule "ETC.3.0: Skip logos (non vertebrate study)" +rule "ETC.3.0: Redact logos" + when + $logo: Image(imageType == ImageType.LOGO) + then + $logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + end + +rule "ETC.3.1: Skip logos (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.skip("ETC.3.0", "Logo Found"); + $logo.skip("ETC.3.1", "Logo Found"); end -rule "ETC.3.1: Redact logos (vertebrate study)" +rule "ETC.3.2: Redact logos (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -1609,7 +1809,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" retract($other); end -// Rule unit: X.9.0 +// Rule unit: X.9 rule "X.9.0: Merge mostly contained signatures" when $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) @@ -1619,7 +1819,7 @@ rule "X.9.0: Merge mostly contained signatures" $signature.addEngine(LayoutEngine.AI); end -// Rule unit: X.10.0 +// Rule unit: X.10 rule "X.10.0: remove false positives of ai" when $anyImage: Image(engines contains LayoutEngine.ALGORITHM) diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index e0cf99ae..214730fe 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -1562,7 +1562,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" retract($other); end -// Rule unit: X.9.0 +// Rule unit: X.9 rule "X.9.0: Merge mostly contained signatures" when $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) @@ -1572,7 +1572,7 @@ rule "X.9.0: Merge mostly contained signatures" $signature.addEngine(LayoutEngine.AI); end -// Rule unit: X.10.0 +// Rule unit: X.10 rule "X.10.0: remove false positives of ai" when $anyImage: Image(engines contains LayoutEngine.ALGORITHM)