Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009 - Document Tree Structure

Browse Source 
* fixed rules to include vertebrate study flags
This commit is contained in:
Kilian Schuettler 2023-05-09 17:20:33 +02:00
parent 0443e21858
commit f0144799c5
1 changed files with 156 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl
View File

@ -66,7 +66,7 @@ rule "0: Recommend CTL/BL laboratory that start with BL or CTL"
end end
// --------------------------------------- CBI Rules ------------------------------------------------------------------- // --------------------------------------- CBI Rules -------------------------------------------------------------------
rule "1: Redact CBI Authors" rule "1: Redact CBI Authors (non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY) $entity: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY)
@ -74,20 +74,31 @@ rule "1: Redact CBI Authors"
$entity.setRedaction(true); $entity.setRedaction(true);
$entity.addMatchedRule(1); $entity.addMatchedRule(1);
$entity.setRedactionReason("Author found"); $entity.setRedactionReason("Author found");
$entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "2: Redact CBI Authors (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
$entity.addMatchedRule(2);
$entity.setRedactionReason("Author found");
$entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); $entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end end
rule "2: Don't redact CBI Address (Non vertebrate study)" rule "3: Don't redact CBI Address (Non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY) $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY)
then then
$entity.setRedaction(false); $entity.setRedaction(false);
$entity.addMatchedRule(2); $entity.addMatchedRule(3);
$entity.setRedactionReason("Address found for non vertebrate study"); $entity.setRedactionReason("Address found for non vertebrate study");
end end
rule "3: Redact CBI Address (Vertebrate study)" rule "4: Redact CBI Address (Vertebrate study)"
when when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY) $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY)
@ -107,8 +118,9 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
insert(falsePositive); insert(falsePositive);
end end
rule "6: Redact all Cell's with Header Author(s) as CBI_author" rule "6: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$table: Table(hasHeader("Author(s)")) $table: Table(hasHeader("Author(s)"))
then then
$table.streamTableCellsWithHeader("Author(s)") $table.streamTableCellsWithHeader("Author(s)")
@ -118,14 +130,32 @@ rule "6: Redact all Cell's with Header Author(s) as CBI_author"
redactionEntity.addMatchedRule(6); redactionEntity.addMatchedRule(6);
redactionEntity.addEngine(Engine.RULE); redactionEntity.addEngine(Engine.RULE);
redactionEntity.setRedactionReason("Author(s) found"); redactionEntity.setRedactionReason("Author(s) found");
redactionEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
end
rule "7: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$table: Table(hasHeader("Author(s)"))
then
$table.streamTableCellsWithHeader("Author(s)")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(7);
redactionEntity.addEngine(Engine.RULE);
redactionEntity.setRedactionReason("Author(s) found");
redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(redactionEntity); insert(redactionEntity);
}); });
end end
rule "8: Redact all Cell's with Header Author(s) as CBI_author" rule "8: Redact all Cell's with Header Author as CBI_author"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$table: Table(hasHeader("Author")) $table: Table(hasHeader("Author"))
then then
$table.streamTableCellsWithHeader("Author") $table.streamTableCellsWithHeader("Author")
@ -135,6 +165,24 @@ rule "8: Redact all Cell's with Header Author(s) as CBI_author"
redactionEntity.addMatchedRule(8); redactionEntity.addMatchedRule(8);
redactionEntity.addEngine(Engine.RULE); redactionEntity.addEngine(Engine.RULE);
redactionEntity.setRedactionReason("Author found"); redactionEntity.setRedactionReason("Author found");
redactionEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
end
rule "9: Redact all Cell's with Header Author as CBI_author"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$table: Table(hasHeader("Author"))
then
$table.streamTableCellsWithHeader("Author")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(9);
redactionEntity.addEngine(Engine.RULE);
redactionEntity.setRedactionReason("Author found");
redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(redactionEntity); insert(redactionEntity);
}); });
@ -142,22 +190,24 @@ rule "8: Redact all Cell's with Header Author(s) as CBI_author"
rule "10: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header" rule "10: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
salience -1
when when
$table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
then then
$table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
end end
rule "14: Add CBI_author with \"et al.\" Regex" rule "14: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsString("et al.")) $section: Section(containsString("et al."))
then then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section) entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> { .forEach(entity -> {
entity.setRedaction(true); entity.setRedaction(true);
entity.setRedactionReason("Author found by \"et al\" regex"); entity.setRedactionReason("Author found by \"et al\" regex");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addMatchedRule(14); entity.addMatchedRule(14);
entity.addEngine(Engine.RULE); entity.addEngine(Engine.RULE);
insert(entity); insert(entity);
@ -165,6 +215,24 @@ rule "14: Add CBI_author with \"et al.\" Regex"
}); });
end end
rule "15: Add CBI_author with \"et al.\" Regex (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> {
entity.setRedaction(true);
entity.setRedactionReason("Author found by \"et al\" regex");
entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addMatchedRule(15);
entity.addEngine(Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
end
rule "16: Add recommendation for Addresses in Test Organism sections" rule "16: Add recommendation for Addresses in Test Organism sections"
when when
$section: Section(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) $section: Section(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
@ -221,8 +289,9 @@ rule "18.1: Do not redact Names and Addresses if published information found in
// --------------------------------------- PII rules ------------------------------------------------------------------- // --------------------------------------- PII rules -------------------------------------------------------------------
rule "19: Redact all PII" rule "19: Redact all PII (non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: RedactionEntity(type == "PII", redaction == false) $pii: RedactionEntity(type == "PII", redaction == false)
then then
$pii.setRedaction(true); $pii.setRedaction(true);
@ -231,8 +300,20 @@ rule "19: Redact all PII"
$pii.addMatchedRule(19); $pii.addMatchedRule(19);
end end
rule "20: Redact all PII (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: RedactionEntity(type == "PII", redaction == false)
then
$pii.setRedaction(true);
$pii.setRedactionReason("Personal Information found");
$pii.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
$pii.addMatchedRule(20);
end
rule "21: Redact Emails by RegEx (Non vertebrate study)" rule "21: Redact Emails by RegEx (Non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsString("@")) $section: Section(containsString("@"))
then then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
@ -245,9 +326,25 @@ rule "21: Redact Emails by RegEx (Non vertebrate study)"
}); });
end end
rule "22: Redact Emails by RegEx (vertebrate study)"
rule "25: Redact Phone and Fax by RegEx"
when when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.setRedaction(true);
emailEntity.setRedactionReason("Found by Email Regex");
emailEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
emailEntity.addMatchedRule(22);
insert(emailEntity);
});
end
rule "25: Redact Phone and Fax by RegEx (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsString("Contact") || $section: Section(containsString("Contact") ||
containsString("Telephone") || containsString("Telephone") ||
containsString("Phone") || containsString("Phone") ||
@ -264,13 +361,14 @@ rule "25: Redact Phone and Fax by RegEx"
contactEntity.setRedaction(true); contactEntity.setRedaction(true);
contactEntity.setRedactionReason("Found by Email Regex"); contactEntity.setRedactionReason("Found by Email Regex");
contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
contactEntity.addMatchedRule(26); contactEntity.addMatchedRule(25);
insert(contactEntity); insert(contactEntity);
}); });
end end
rule "25: Redact Phone and Fax by RegEx" rule "26: Redact Phone and Fax by RegEx (vertebrate study)"
when when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsString("Contact") || $section: Section(containsString("Contact") ||
containsString("Telephone") || containsString("Telephone") ||
containsString("Phone") || containsString("Phone") ||
@ -286,15 +384,16 @@ rule "25: Redact Phone and Fax by RegEx"
.forEach(contactEntity -> { .forEach(contactEntity -> {
contactEntity.setRedaction(true); contactEntity.setRedaction(true);
contactEntity.setRedactionReason("Found by Email Regex"); contactEntity.setRedactionReason("Found by Email Regex");
contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); contactEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
contactEntity.addMatchedRule(26); contactEntity.addMatchedRule(26);
insert(contactEntity); insert(contactEntity);
}); });
end end
rule "27: Redact AUTHOR(S)" rule "27: Redact AUTHOR(S) (non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
@ -302,14 +401,30 @@ rule "27: Redact AUTHOR(S)"
authorEntity.setRedaction(true); authorEntity.setRedaction(true);
authorEntity.addMatchedRule(27); authorEntity.addMatchedRule(27);
authorEntity.setRedactionReason("AUTHOR(S) was found"); authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
end
rule "28: Redact AUTHOR(S) (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(28);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(authorEntity); insert(authorEntity);
}); });
end end
rule "29: Redact AUTHOR(S)" rule "29: Redact AUTHOR(S) (non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
@ -317,12 +432,28 @@ rule "29: Redact AUTHOR(S)"
authorEntity.setRedaction(true); authorEntity.setRedaction(true);
authorEntity.addMatchedRule(29); authorEntity.addMatchedRule(29);
authorEntity.setRedactionReason("AUTHOR(S) was found"); authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
end
rule "30: Redact AUTHOR(S) (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(30);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(authorEntity); insert(authorEntity);
}); });
end end
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when when
not FileAttribute(label == "Vertebrate Study", value == "Yes") not FileAttribute(label == "Vertebrate Study", value == "Yes")
$section: Section(excludesTables, containsString("PERFORMING LABORATORY:")) $section: Section(excludesTables, containsString("PERFORMING LABORATORY:"))
@ -338,6 +469,7 @@ rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
end end
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when when
FileAttribute(label == "Vertebrate Study", value == "Yes") FileAttribute(label == "Vertebrate Study", value == "Yes")
$section: Section(excludesTables, containsString("PERFORMING LABORATORY:")) $section: Section(excludesTables, containsString("PERFORMING LABORATORY:"))
@ -368,7 +500,7 @@ rule "33: Purity Hint"
rule "34: Redact signatures (not Vertebrate Study)" rule "34: Redact signatures (not Vertebrate Study)"
when when
not FileAttribute(label == "Vertebrate Study", value == "Yes") not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$signature: Image(imageType == ImageType.SIGNATURE) $signature: Image(imageType == ImageType.SIGNATURE)
then then
$signature.setRedaction(true); $signature.setRedaction(true);
@ -379,18 +511,19 @@ rule "34: Redact signatures (not Vertebrate Study)"
rule "35: Redact signatures (Vertebrate Study)" rule "35: Redact signatures (Vertebrate Study)"
when when
FileAttribute(label == "Vertebrate Study", value == "Yes") FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$signature: Image(imageType == ImageType.SIGNATURE) $signature: Image(imageType == ImageType.SIGNATURE)
then then
$signature.setRedaction(true); $signature.setRedaction(true);
$signature.setMatchedRule(35); $signature.setMatchedRule(35);
$signature.setRedactionReason("Signature Found"); $signature.setRedactionReason("Signature Found");
$signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); $signature.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end end
rule "36: Redact logos" rule "36: Redact logos"
when when
not FileAttribute(label == "Vertbrate Study", value.toLowerCase() == "yes")
$logo: Image(imageType == ImageType.LOGO) $logo: Image(imageType == ImageType.LOGO)
then then
$logo.setRedaction(true); $logo.setRedaction(true);