From f0144799c50e8a202d6886865bbc6bd575018617 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 9 May 2023 17:20:33 +0200 Subject: [PATCH] RED-6009 - Document Tree Structure * fixed rules to include vertebrate study flags --- .../resources/drools/prod_syngenta_new.drl | 179 +++++++++++++++--- 1 file changed, 156 insertions(+), 23 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl index a14f2480..653961a5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl @@ -66,7 +66,7 @@ rule "0: Recommend CTL/BL laboratory that start with BL or CTL" end // --------------------------------------- CBI Rules ------------------------------------------------------------------- -rule "1: Redact CBI Authors" +rule "1: Redact CBI Authors (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY) @@ -74,20 +74,31 @@ rule "1: Redact CBI Authors" $entity.setRedaction(true); $entity.addMatchedRule(1); $entity.setRedactionReason("Author found"); + $entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "2: Redact CBI Authors (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY) + then + $entity.setRedaction(true); + $entity.addMatchedRule(2); + $entity.setRedactionReason("Author found"); $entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); end -rule "2: Don't redact CBI Address (Non vertebrate study)" +rule "3: Don't redact CBI Address (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY) then $entity.setRedaction(false); - $entity.addMatchedRule(2); + $entity.addMatchedRule(3); $entity.setRedactionReason("Address found for non vertebrate study"); end -rule "3: Redact CBI Address (Vertebrate study)" +rule "4: Redact CBI Address (Vertebrate study)" when FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY) @@ -107,8 +118,9 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" insert(falsePositive); end -rule "6: Redact all Cell's with Header Author(s) as CBI_author" +rule "6: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)" when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $table: Table(hasHeader("Author(s)")) then $table.streamTableCellsWithHeader("Author(s)") @@ -118,14 +130,32 @@ rule "6: Redact all Cell's with Header Author(s) as CBI_author" redactionEntity.addMatchedRule(6); redactionEntity.addEngine(Engine.RULE); redactionEntity.setRedactionReason("Author(s) found"); + redactionEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + insert(redactionEntity); + }); + end + +rule "7: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)")) + then + $table.streamTableCellsWithHeader("Author(s)") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.addMatchedRule(7); + redactionEntity.addEngine(Engine.RULE); + redactionEntity.setRedactionReason("Author(s) found"); redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); insert(redactionEntity); }); end -rule "8: Redact all Cell's with Header Author(s) as CBI_author" +rule "8: Redact all Cell's with Header Author as CBI_author" agenda-group "LOCAL_DICTIONARY_ADDS" when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $table: Table(hasHeader("Author")) then $table.streamTableCellsWithHeader("Author") @@ -135,6 +165,24 @@ rule "8: Redact all Cell's with Header Author(s) as CBI_author" redactionEntity.addMatchedRule(8); redactionEntity.addEngine(Engine.RULE); redactionEntity.setRedactionReason("Author found"); + redactionEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + insert(redactionEntity); + }); + end + +rule "9: Redact all Cell's with Header Author as CBI_author" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author")) + then + $table.streamTableCellsWithHeader("Author") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.addMatchedRule(9); + redactionEntity.addEngine(Engine.RULE); + redactionEntity.setRedactionReason("Author found"); redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); insert(redactionEntity); }); @@ -142,22 +190,24 @@ rule "8: Redact all Cell's with Header Author(s) as CBI_author" rule "10: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header" agenda-group "LOCAL_DICTIONARY_ADDS" + salience -1 when $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) then $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); end -rule "14: Add CBI_author with \"et al.\" Regex" +rule "14: Add CBI_author with \"et al.\" Regex (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - $section: Section(containsString("et al.")) + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("et al.")) then - entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section) + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { entity.setRedaction(true); entity.setRedactionReason("Author found by \"et al\" regex"); - entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); entity.addMatchedRule(14); entity.addEngine(Engine.RULE); insert(entity); @@ -165,6 +215,24 @@ rule "14: Add CBI_author with \"et al.\" Regex" }); end +rule "15: Add CBI_author with \"et al.\" Regex (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section) + .forEach(entity -> { + entity.setRedaction(true); + entity.setRedactionReason("Author found by \"et al\" regex"); + entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.addMatchedRule(15); + entity.addEngine(Engine.RULE); + insert(entity); + dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); + }); + end + rule "16: Add recommendation for Addresses in Test Organism sections" when $section: Section(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) @@ -221,8 +289,9 @@ rule "18.1: Do not redact Names and Addresses if published information found in // --------------------------------------- PII rules ------------------------------------------------------------------- -rule "19: Redact all PII" +rule "19: Redact all PII (non vertebrate study)" when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $pii: RedactionEntity(type == "PII", redaction == false) then $pii.setRedaction(true); @@ -231,8 +300,20 @@ rule "19: Redact all PII" $pii.addMatchedRule(19); end +rule "20: Redact all PII (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $pii: RedactionEntity(type == "PII", redaction == false) + then + $pii.setRedaction(true); + $pii.setRedactionReason("Personal Information found"); + $pii.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.addMatchedRule(20); + end + rule "21: Redact Emails by RegEx (Non vertebrate study)" when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -245,9 +326,25 @@ rule "21: Redact Emails by RegEx (Non vertebrate study)" }); end - -rule "25: Redact Phone and Fax by RegEx" +rule "22: Redact Emails by RegEx (vertebrate study)" when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> { + emailEntity.setRedaction(true); + emailEntity.setRedactionReason("Found by Email Regex"); + emailEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + emailEntity.addMatchedRule(22); + insert(emailEntity); + }); + end + + +rule "25: Redact Phone and Fax by RegEx (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -264,13 +361,14 @@ rule "25: Redact Phone and Fax by RegEx" contactEntity.setRedaction(true); contactEntity.setRedactionReason("Found by Email Regex"); contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); - contactEntity.addMatchedRule(26); + contactEntity.addMatchedRule(25); insert(contactEntity); }); end -rule "25: Redact Phone and Fax by RegEx" +rule "26: Redact Phone and Fax by RegEx (vertebrate study)" when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -286,15 +384,16 @@ rule "25: Redact Phone and Fax by RegEx" .forEach(contactEntity -> { contactEntity.setRedaction(true); contactEntity.setRedactionReason("Found by Email Regex"); - contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + contactEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); contactEntity.addMatchedRule(26); insert(contactEntity); }); end -rule "27: Redact AUTHOR(S)" +rule "27: Redact AUTHOR(S) (non vertebrate study)" when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) @@ -302,14 +401,30 @@ rule "27: Redact AUTHOR(S)" authorEntity.setRedaction(true); authorEntity.addMatchedRule(27); authorEntity.setRedactionReason("AUTHOR(S) was found"); - authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + insert(authorEntity); + }); + end + +rule "28: Redact AUTHOR(S) (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.addMatchedRule(28); + authorEntity.setRedactionReason("AUTHOR(S) was found"); + authorEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); insert(authorEntity); }); end -rule "29: Redact AUTHOR(S)" +rule "29: Redact AUTHOR(S) (non vertebrate study)" when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) @@ -317,12 +432,28 @@ rule "29: Redact AUTHOR(S)" authorEntity.setRedaction(true); authorEntity.addMatchedRule(29); authorEntity.setRedactionReason("AUTHOR(S) was found"); - authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + insert(authorEntity); + }); + end + +rule "30: Redact AUTHOR(S) (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.addMatchedRule(30); + authorEntity.setRedactionReason("AUTHOR(S) was found"); + authorEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); insert(authorEntity); }); end rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when not FileAttribute(label == "Vertebrate Study", value == "Yes") $section: Section(excludesTables, containsString("PERFORMING LABORATORY:")) @@ -338,6 +469,7 @@ rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" end rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study", value == "Yes") $section: Section(excludesTables, containsString("PERFORMING LABORATORY:")) @@ -368,7 +500,7 @@ rule "33: Purity Hint" rule "34: Redact signatures (not Vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.setRedaction(true); @@ -379,18 +511,19 @@ rule "34: Redact signatures (not Vertebrate Study)" rule "35: Redact signatures (Vertebrate Study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.setRedaction(true); $signature.setMatchedRule(35); $signature.setRedactionReason("Signature Found"); - $signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); end rule "36: Redact logos" when + not FileAttribute(label == "Vertbrate Study", value.toLowerCase() == "yes") $logo: Image(imageType == ImageType.LOGO) then $logo.setRedaction(true);