5c4c8218f5
* added a test for removing entities by ManualResizeRedaction in ReAnalysis * Fixed Set logic when removing and readding a resized entity * Refactored RedactionLog change computation
674 lines
31 KiB
Plaintext
674 lines
31 KiB
Plaintext
package drools
|
|
|
|
import static java.lang.String.format;
|
|
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.anyMatch;
|
|
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.exactMatch;
|
|
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
|
|
|
|
import java.util.List;
|
|
import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
|
|
import java.util.LinkedList;
|
|
import java.util.HashSet;
|
|
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*
|
|
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
|
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
|
import java.util.Set
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
|
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
|
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
|
|
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
|
|
import java.util.stream.Collectors
|
|
import java.util.Collection
|
|
import java.util.stream.Stream
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
|
|
|
|
global DocumentGraph document
|
|
global EntityCreationService entityCreationService
|
|
global ManualRedactionApplicationService manualRedactionApplicationService
|
|
global Dictionary dictionary
|
|
|
|
// --------------------------------------- queries -------------------------------------------------------------------
|
|
|
|
query "getFileAttributes"
|
|
$fileAttribute: FileAttribute()
|
|
end
|
|
|
|
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
|
|
|
|
rule "add NER Entities of type CBI_author or CBI_address"
|
|
salience 999
|
|
when
|
|
$nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address"))
|
|
then
|
|
RedactionEntity redactionEntity = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document);
|
|
redactionEntity.addEngine(Engine.NER);
|
|
insert(redactionEntity);
|
|
end
|
|
|
|
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
|
|
|
rule "0: Expand CBI_author entities with firstname initials"
|
|
no-loop true
|
|
when
|
|
$entityToExpand: RedactionEntity(type == "CBI_author",
|
|
value.matches("[^\\s]+"),
|
|
textAfter.startsWith(" "),
|
|
anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
|
|
)
|
|
then
|
|
RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)");
|
|
expandedEntity.addMatchedRule(0);
|
|
$entityToExpand.removeFromGraph();
|
|
retract($entityToExpand);
|
|
insert(expandedEntity);
|
|
end
|
|
|
|
rule "0: Expand CBI_author and PII entities with salutation prefix"
|
|
when
|
|
$entityToExpand: RedactionEntity((type == "CBI_author" || type == "PII"), anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
|
|
then
|
|
RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
|
|
expandedEntity.addMatchedRule(0);
|
|
insert(expandedEntity);
|
|
end
|
|
|
|
rule "1: Redacted because Section contains Vertebrate"
|
|
when
|
|
$section: SectionNode(hasEntitiesOfType("vertebrate"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(true);
|
|
redactionEntity.addMatchedRule(1);
|
|
redactionEntity.setRedactionReason("Vertebrate Found in this section");
|
|
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
});
|
|
end
|
|
|
|
rule "2: Not Redacted because Section contains no Vertebrate"
|
|
when
|
|
$section: SectionNode(!hasEntitiesOfType("vertebrate"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(false);
|
|
redactionEntity.addMatchedRule(2);
|
|
redactionEntity.setRedactionReason("No Vertebrate Found in this section");
|
|
});
|
|
end
|
|
|
|
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
|
|
when
|
|
$section: SectionNode(hasEntitiesOfType("vertebrate"),
|
|
hasEntitiesOfType("no_redaction_indicator"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(false);
|
|
redactionEntity.addMatchedRule(3);
|
|
redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator found in this section");
|
|
});
|
|
end
|
|
|
|
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
|
|
when
|
|
$section: SectionNode(hasEntitiesOfType("vertebrate"),
|
|
hasEntitiesOfType("no_redaction_indicator"),
|
|
hasEntitiesOfType("redaction_indicator"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(true);
|
|
redactionEntity.addMatchedRule(4);
|
|
redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator, but also redaction-indicator, found in this section");
|
|
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
});
|
|
end
|
|
|
|
rule "5: Do not redact Names and Addresses if published information found"
|
|
|
|
when
|
|
$section: SectionNode(hasEntitiesOfType("vertebrate"),
|
|
hasEntitiesOfType("published_information"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
List<RedactionEntity> publishedInformationEntities = $section.getEntitiesOfType("published_information");
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(false);
|
|
redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section");
|
|
redactionEntity.addReferences(publishedInformationEntities);
|
|
});
|
|
end
|
|
|
|
rule "6.0: Add all Cell's with Header Author(s) as CBI_author"
|
|
when
|
|
$table: TableNode(hasHeader("Author(s)"))
|
|
then
|
|
$table.streamTableCellsWithHeader("Author(s)")
|
|
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.addMatchedRule(6);
|
|
redactionEntity.setRedactionReason("Author(s) header found");
|
|
insert(redactionEntity);
|
|
});
|
|
end
|
|
|
|
rule "6.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
|
|
when
|
|
$table: TableNode(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No"))
|
|
then
|
|
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
|
|
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
|
|
.forEach(authorEntity -> {
|
|
authorEntity.setRedaction(false);
|
|
authorEntity.setRedactionReason("Not redacted because it's row does not belong to a vertebrate study");
|
|
authorEntity.setLegalBasis("");
|
|
authorEntity.addMatchedRule(6);
|
|
});
|
|
end
|
|
|
|
rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
|
|
when
|
|
$table: TableNode(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes"))
|
|
then
|
|
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
|
|
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
|
|
.forEach(authorEntity -> {
|
|
authorEntity.setRedaction(true);
|
|
authorEntity.setRedactionReason("Redacted because it's row belongs to a vertebrate study");
|
|
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
authorEntity.addMatchedRule(7);
|
|
});
|
|
end
|
|
|
|
rule "8: Redact if must_redact entity is found"
|
|
when
|
|
$section: SectionNode(hasEntitiesOfType("must_redact"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(true);
|
|
redactionEntity.setRedactionReason("must_redact entry was found.");
|
|
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
redactionEntity.addMatchedRule(8);
|
|
});
|
|
end
|
|
|
|
rule "9: Redact CBI_sponsor entities if preceded by \" batches produced at\""
|
|
when
|
|
$sponsorEntity: RedactionEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
|
|
then
|
|
$sponsorEntity.setRedaction(true);
|
|
$sponsorEntity.setRedactionReason("Redacted because it represents a sponsor company");
|
|
$sponsorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
$sponsorEntity.addMatchedRule(9);
|
|
end
|
|
|
|
rule "10: Redact row if row contains \"determination of residues\" and livestock keyword"
|
|
when
|
|
$keyword: String() from List.of("livestock",
|
|
"live stock",
|
|
"tissue",
|
|
"tissues",
|
|
"liver",
|
|
"muscle",
|
|
"bovine",
|
|
"ruminant",
|
|
"ruminants")
|
|
$residueKeyword: String() from List.of("determination of residues", "determination of total residues")
|
|
$table: TableNode(containsStringIgnoreCase($residueKeyword)
|
|
&& containsStringIgnoreCase($keyword))
|
|
then
|
|
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
|
|
.forEach(keywordEntity -> insert(keywordEntity));
|
|
|
|
$table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
|
|
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(true);
|
|
redactionEntity.setRedactionReason("Determination of residues and keyword \"" + $keyword + "\" was found.");
|
|
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
redactionEntity.addMatchedRule(10);
|
|
});
|
|
end
|
|
|
|
rule "11: Redact if CTL/* or BL/* was found"
|
|
when
|
|
$section: SectionNode(excludesTables, (containsString("CTL/") || containsString("BL/")))
|
|
then
|
|
entityCreationService.byString("CTL/", "must_redact", EntityType.ENTITY, $section)
|
|
.forEach(mustRedactEntity -> insert(mustRedactEntity));
|
|
entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section)
|
|
.forEach(mustRedactEntity -> insert(mustRedactEntity));
|
|
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(true);
|
|
redactionEntity.setRedactionReason("Laboratory for vertebrate studies found");
|
|
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
redactionEntity.addMatchedRule(11);
|
|
});
|
|
end
|
|
|
|
rule "12: Add CBI_author with \"et al.\" Regex"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
$section: SectionNode(containsString("et al."))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
|
|
.forEach(entity -> {
|
|
entity.setRedaction(true);
|
|
entity.setRedactionReason("Author found by \"et al\" regex");
|
|
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
entity.addMatchedRule(12);
|
|
insert(entity);
|
|
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
|
|
});
|
|
end
|
|
|
|
rule "13: Add recommendation for Addresses in Test Organism sections"
|
|
when
|
|
$section: SectionNode(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
|
|
then
|
|
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedactionReason("Line after \"Source\" in Test Organism Section");
|
|
redactionEntity.addMatchedRule(13);
|
|
insert(redactionEntity);
|
|
});
|
|
end
|
|
|
|
rule "14: Add recommendation for Addresses in Test Animals sections"
|
|
|
|
when
|
|
$section: SectionNode(excludesTables, containsString("Species:"), containsString("Source:"))
|
|
then
|
|
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedactionReason("Line after \"Source:\" in Test Animals Section");
|
|
redactionEntity.addMatchedRule(14);
|
|
insert(redactionEntity);
|
|
});
|
|
end
|
|
|
|
// --------------------------------------- PII rules -------------------------------------------------------------------
|
|
|
|
rule "15: Redact all PII"
|
|
when
|
|
$pii: RedactionEntity(type == "PII", redaction == false)
|
|
then
|
|
$pii.setRedaction(true);
|
|
$pii.setRedactionReason("PII found");
|
|
$pii.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
$pii.addMatchedRule(15);
|
|
end
|
|
|
|
rule "16: Redact Emails by RegEx (Non vertebrate study)"
|
|
when
|
|
$section: SectionNode(containsString("@"))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section)
|
|
.forEach(emailEntity -> {
|
|
emailEntity.setRedaction(true);
|
|
emailEntity.setRedactionReason("Found by Email Regex");
|
|
emailEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
emailEntity.addMatchedRule(16);
|
|
insert(emailEntity);
|
|
});
|
|
end
|
|
|
|
rule "17: Redact line after contact information keywords"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
$contactKeyword: String() from List.of("Contact point:",
|
|
"Contact:",
|
|
"Alternative contact:",
|
|
"European contact:",
|
|
"No:",
|
|
"Contact:",
|
|
"Tel.:",
|
|
"Tel:",
|
|
"Telephone number:",
|
|
"Telephone No:",
|
|
"Telephone:",
|
|
"Phone No.",
|
|
"Phone:",
|
|
"Fax number:",
|
|
"Fax:",
|
|
"E-mail:",
|
|
"Email:",
|
|
"e-mail:",
|
|
"E-mail address:")
|
|
$section: SectionNode(excludesTables, containsString($contactKeyword))
|
|
then
|
|
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
|
|
.forEach(contactEntity -> {
|
|
contactEntity.setRedaction(true);
|
|
contactEntity.addMatchedRule(17);
|
|
contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword");
|
|
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
insert(contactEntity);
|
|
dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false);
|
|
});
|
|
end
|
|
|
|
|
|
rule "18: redact line between contact keywords"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
|
|
then
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
)
|
|
.forEach(contactEntity -> {
|
|
contactEntity.setRedaction(true);
|
|
contactEntity.addMatchedRule(18);
|
|
contactEntity.setRedactionReason("Found between contact keywords");
|
|
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
insert(contactEntity);
|
|
dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false);
|
|
});
|
|
end
|
|
|
|
rule "19: Redact AUTHOR(S)"
|
|
when
|
|
FileAttribute(placeholder == "{fileattributes.vertebrateStudy}", value == "true")
|
|
$section: SectionNode(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"))
|
|
then
|
|
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
|
|
.forEach(authorEntity -> {
|
|
authorEntity.setRedaction(true);
|
|
authorEntity.addMatchedRule(19);
|
|
authorEntity.setRedactionReason("AUTHOR(S) was found");
|
|
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
insert(authorEntity);
|
|
});
|
|
end
|
|
|
|
rule "20: Redact PERFORMING LABORATORY"
|
|
when
|
|
$section: SectionNode(excludesTables, containsString("PERFORMING LABORATORY:"))
|
|
then
|
|
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
|
|
.forEach(authorEntity -> {
|
|
authorEntity.setRedaction(true);
|
|
authorEntity.addMatchedRule(20);
|
|
authorEntity.setRedactionReason("PERFORMING LABORATORY was found");
|
|
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
insert(authorEntity);
|
|
});
|
|
end
|
|
|
|
rule "21: Redact On behalf of Sequani Ltd.:"
|
|
when
|
|
$section: SectionNode(excludesTables, containsString("On behalf of Sequani Ltd.: Name Title"))
|
|
then
|
|
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
|
|
.forEach(authorEntity -> {
|
|
authorEntity.setRedaction(true);
|
|
authorEntity.addMatchedRule(21);
|
|
authorEntity.setRedactionReason("On behalf of Sequani Ltd.: Name Title was found");
|
|
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
insert(authorEntity);
|
|
});
|
|
end
|
|
|
|
rule "22: Redact On behalf of Syngenta Ltd.:"
|
|
when
|
|
$section: SectionNode(excludesTables, containsString("On behalf of Syngenta Ltd.: Name Title"))
|
|
then
|
|
entityCreationService.betweenStrings("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", EntityType.ENTITY, $section)
|
|
.forEach(authorEntity -> {
|
|
authorEntity.setRedaction(true);
|
|
authorEntity.addMatchedRule(21);
|
|
authorEntity.setRedactionReason("On behalf of Syngenta Ltd.: Name Title was found");
|
|
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
|
|
insert(authorEntity);
|
|
});
|
|
end
|
|
|
|
rule "26: Redact signatures"
|
|
when
|
|
$signature: ImageNode(imageType == ImageType.SIGNATURE)
|
|
then
|
|
$signature.setRedaction(true);
|
|
$signature.setMatchedRule(26);
|
|
$signature.setRedactionReason("Signature Found");
|
|
$signature.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
end
|
|
|
|
rule "27: Redact formulas"
|
|
when
|
|
$formula: ImageNode(imageType == ImageType.FORMULA)
|
|
then
|
|
$formula.setRedaction(true);
|
|
$formula.setMatchedRule(27);
|
|
$formula.setRedactionReason("Formula Found");
|
|
$formula.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
end
|
|
|
|
rule "28: Redact logos"
|
|
when
|
|
$logo: ImageNode(imageType == ImageType.LOGO)
|
|
then
|
|
$logo.setRedaction(true);
|
|
$logo.setMatchedRule(28);
|
|
$logo.setRedactionReason("Logo Found");
|
|
$logo.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
end
|
|
|
|
rule "29: Redact Dossier Redactions"
|
|
when
|
|
$dossierRedaction: RedactionEntity(type == "dossier_redactions")
|
|
then
|
|
$dossierRedaction.setRedaction(true);
|
|
$dossierRedaction.addMatchedRule(29);
|
|
$dossierRedaction.setRedactionReason("Dossier Redaction found");
|
|
$dossierRedaction.setLegalBasis("Article 39(1)(2) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
rule "30: Remove Dossier redactions if file is confidential"
|
|
when
|
|
FileAttribute(label == "Confidentiality", value == "confidential")
|
|
$dossierRedaction: RedactionEntity(type == "dossier_redactions")
|
|
then
|
|
$dossierRedaction.removeFromGraph();
|
|
retract($dossierRedaction)
|
|
end
|
|
|
|
rule "101: Redact CAS Number"
|
|
when
|
|
$table: TableNode(hasHeader("Sample #"))
|
|
then
|
|
$table.streamTableCellsWithHeader("Sample #")
|
|
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
|
|
.forEach(redactionEntity -> {
|
|
redactionEntity.setRedaction(true);
|
|
redactionEntity.addMatchedRule(101);
|
|
redactionEntity.setRedactionReason("Sample # found in Header");
|
|
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
insert(redactionEntity);
|
|
});
|
|
end
|
|
|
|
rule "102: Guidelines FileAttributes"
|
|
when
|
|
$section: SectionNode(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
|
|
then
|
|
RedactionSearchUtils.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.buildTextBlock()).stream()
|
|
.map(boundary -> $section.buildTextBlock().subSequence(boundary).toString())
|
|
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
|
|
.forEach(fileAttribute -> insert(fileAttribute));
|
|
end
|
|
|
|
// --------------------------------------- manual redaction rules -------------------------------------------------------------------
|
|
|
|
rule "Apply manual resize redaction"
|
|
salience 128
|
|
when
|
|
$resizeRedaction: ManualResizeRedaction($id: annotationId)
|
|
$entityToBeResized: RedactionEntity(matchesAnnotationId($id))
|
|
then
|
|
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
|
|
retract($resizeRedaction);
|
|
update($entityToBeResized);
|
|
end
|
|
|
|
rule "Apply id removals that are valid and not in forced redactions to Entity"
|
|
salience 128
|
|
when
|
|
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
|
|
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
|
|
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToBeRemoved.removeFromGraph();
|
|
retract($entityToBeRemoved);
|
|
end
|
|
|
|
rule "Apply id removals that are valid and not in forced redactions to Image"
|
|
salience 128
|
|
when
|
|
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
|
|
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
|
|
$entityToBeRemoved: ImageNode($id == id)
|
|
then
|
|
$entityToBeRemoved.setIgnored(true);
|
|
end
|
|
|
|
rule "Apply force redaction"
|
|
salience 128
|
|
when
|
|
ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
|
|
$entityToForce: RedactionEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToForce.setLegalBasis($legalBasis);
|
|
$entityToForce.setRedaction(true);
|
|
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
|
|
end
|
|
|
|
rule "Apply image recategorization"
|
|
salience 128
|
|
when
|
|
ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
|
|
$image: ImageNode($id == id)
|
|
then
|
|
$image.setImageType(parseImageType($imageType));
|
|
end
|
|
|
|
// --------------------------------------- merging rules -------------------------------------------------------------------
|
|
|
|
rule "merge intersecting Entities of same type"
|
|
salience 64
|
|
when
|
|
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$first.removeFromGraph();
|
|
$second.removeFromGraph();
|
|
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
|
|
retract($first);
|
|
retract($second);
|
|
insert(mergedEntity);
|
|
end
|
|
|
|
rule "remove Entity contained by Entity of same type"
|
|
salience 64
|
|
when
|
|
$larger: RedactionEntity($type: type, $entityType: entityType)
|
|
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$contained.removeFromGraph();
|
|
retract($contained);
|
|
end
|
|
|
|
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
|
salience 64
|
|
when
|
|
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
|
|
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$entity.removeFromGraph();
|
|
retract($entity)
|
|
end
|
|
|
|
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
|
|
salience 64
|
|
when
|
|
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
|
|
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$recommendation.removeFromGraph();
|
|
retract($recommendation);
|
|
end
|
|
|
|
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
|
|
salience 64
|
|
when
|
|
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
|
|
$recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$recommendation.removeFromGraph();
|
|
retract($recommendation);
|
|
end
|
|
|
|
rule "remove Entity of lower rank, when equal boundaries and entityType"
|
|
salience 32
|
|
when
|
|
$higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
|
|
$lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
|
|
then
|
|
$lowerRank.removeFromGraph();
|
|
retract($lowerRank);
|
|
end
|
|
|
|
// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
|
|
|
|
rule "remove duplicate FileAttributes"
|
|
salience 64
|
|
when
|
|
$first: FileAttribute($label: label, $value: value)
|
|
$second: FileAttribute(this != $first, label == $label, value == $value)
|
|
then
|
|
retract($second);
|
|
end
|
|
|
|
// --------------------------------------- local dictionary search -------------------------------------------------------------------
|
|
|
|
rule "run local dictionary search"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
salience -999
|
|
when
|
|
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
|
|
then
|
|
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
|
|
.forEach(entity -> {
|
|
entity.addEngine(Engine.RULE);
|
|
insert(entity);
|
|
});
|
|
end
|