@ -0,0 +1,561 @@
package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
import java.util.List;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
import java.util.LinkedList;
import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
global DocumentGraph document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
// --------------------------------------- Syngenta specific laboratory recommendation -------------------------------------------------------------------
rule "0: Recommend CTL/BL laboratory that start with BL or CTL"
when
$section: SectionNode(containsString("CT") || containsString("BL"))
then
/* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.addMatchedRule(0);
entity.addEngine(Engine.RULE);
insert(entity);
});
end
// --------------------------------------- CBI Rules -------------------------------------------------------------------
rule "1: Redact CBI Authors"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
$entity.addMatchedRule(1);
$entity.setRedactionReason("Author found");
$entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "2: Don't redact CBI Address (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY)
then
$entity.setRedaction(false);
$entity.addMatchedRule(2);
$entity.setRedactionReason("Address found for non vertebrate study");
end
rule "3: Redact CBI Address (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
$entity.addMatchedRule(4);
$entity.setRedactionReason("Address found");
$entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
when
$entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’ ’ ʼ ˈ ´ ‘ ′ ʻ ’
then
RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
falsePositive.addMatchedRule(5);
insert(falsePositive);
end
rule "6: Redact all Cell's with Header Author(s) as CBI_author"
when
$table: TableNode(hasHeader("Author(s)"))
then
$table.streamTableCellsWithHeader("Author(s)")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(6);
redactionEntity.addEngine(Engine.RULE);
redactionEntity.setRedactionReason("Author(s) found");
redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
end
rule "8: Redact all Cell's with Header Author(s) as CBI_author"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$table: TableNode(hasHeader("Author"))
then
$table.streamTableCellsWithHeader("Author")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(8);
redactionEntity.addEngine(Engine.RULE);
redactionEntity.setRedactionReason("Author found");
redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
end
rule "10: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$table: TableNode(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
then
$table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
end
rule "14: Add CBI_author with \"et al.\" Regex"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: SectionNode(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> {
entity.setRedaction(true);
entity.setRedactionReason("Author found by \"et al\" regex");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.addMatchedRule(14);
entity.addEngine(Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
end
rule "16: Add recommendation for Addresses in Test Organism sections"
when
$section: SectionNode(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.setRedactionReason("Line after \"Source\" in Test Organism Section");
entity.addEngine(Engine.RULE);
entity.addMatchedRule(16);
insert(entity);
});
end
rule "17: Add recommendation for Addresses in Test Animals sections"
when
$section: SectionNode(excludesTables, containsString("Species:"), containsString("Source:"))
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.setRedactionReason("Line after \"Source:\" in Test Animals Section");
entity.addEngine(Engine.RULE);
entity.addMatchedRule(17);
insert(entity);
});
end
rule "18: Do not redact Names and Addresses if published information found in section without tables"
when
$section: SectionNode(excludesTables &&
hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
List<RedactionEntity> publishedInformationEntities = $section.getEntitiesOfType("published_information");
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.setRedactionReason("Published Information found");
redactionEntity.addReferences(publishedInformationEntities);
});
end
rule "18.0: Do not redact Names and Addresses if published information found in same table row"
when
$table: TableNode(hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.setRedactionReason("Published Information found in row");
redactionEntity.addReferences($table.getEntitiesOfTypeInSameRow("published_information", redactionEntity));
});
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "19: Redact all PII"
when
$pii: RedactionEntity(type == "PII", redaction == false)
then
$pii.setRedaction(true);
$pii.setRedactionReason("Personal Information found");
$pii.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
$pii.addMatchedRule(19);
end
rule "21: Redact Emails by RegEx (Non vertebrate study)"
when
$section: SectionNode(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.setRedaction(true);
emailEntity.setRedactionReason("Found by Email Regex");
emailEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
emailEntity.addMatchedRule(21);
insert(emailEntity);
});
end
rule "25: Redact Phone and Fax by RegEx"
when
$section: SectionNode(containsString("Contact") ||
containsString("Telephone") ||
containsString("Phone") ||
containsString("Ph.") ||
containsString("Fax") ||
containsString("Tel") ||
containsString("Ter") ||
containsString("Mobile") ||
containsString("Fel") ||
containsString("Fer"))
then
entityCreationService.byRegex("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.setRedactionReason("Found by Email Regex");
contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
contactEntity.addMatchedRule(26);
insert(contactEntity);
});
end
rule "27: Redact AUTHOR(S)"
when
$section: SectionNode(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(27);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "29: Redact AUTHOR(S)"
when
$section: SectionNode(excludesTables, containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(29);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$section: SectionNode(excludesTables, containsString("PERFORMING LABORATORY:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.setRedaction(false);
laboratoryEntity.addMatchedRule(31);
laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$section: SectionNode(excludesTables, containsString("PERFORMING LABORATORY:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.setRedaction(true);
laboratoryEntity.addMatchedRule(32);
laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found");
laboratoryEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "33: Purity Hint"
when
$section: SectionNode(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
.forEach(hint -> {
hint.addEngine(Engine.RULE);
hint.addMatchedRule(33);
});
end
rule "34: Redact signatures (not Vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: ImageNode(imageType == ImageType.SIGNATURE)
then
$signature.setRedaction(true);
$signature.setMatchedRule(34);
$signature.setRedactionReason("Signature Found");
$signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "35: Redact signatures (Vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: ImageNode(imageType == ImageType.SIGNATURE)
then
$signature.setRedaction(true);
$signature.setMatchedRule(35);
$signature.setRedactionReason("Signature Found");
$signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "36: Redact logos"
when
$logo: ImageNode(imageType == ImageType.LOGO)
then
$logo.setRedaction(true);
$logo.setMatchedRule(36);
$logo.setRedactionReason("Logo Found");
$logo.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
end
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
rule "add NER Entities of type CBI_author"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
.forEach(entity -> insert(entity));
end
rule "combine and add NER Entities as CBI_address"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> {
entity.addEngine(Engine.NER);
insert(entity);
});
end
// --------------------------------------- manual redaction rules -------------------------------------------------------------------
rule "Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId)
$entityToBeResized: RedactionEntity(matchesAnnotationId($id))
then
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
end
rule "Apply id removals that are valid and not in forced redactions to Entity"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.removeFromGraph();
retract($entityToBeRemoved);
end
rule "Apply id removals that are valid and not in forced redactions to Image"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: ImageNode($id == id)
then
$entityToBeRemoved.setIgnored(true);
end
rule "Apply force redaction"
salience 128
when
ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$entityToForce: RedactionEntity(matchesAnnotationId($id))
then
$entityToForce.setLegalBasis($legalBasis);
$entityToForce.setRedaction(true);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
end
rule "Apply image recategorization"
salience 128
when
ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
$image: ImageNode($id == id)
then
$image.setImageType(parseImageType($imageType));
end
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "remove Entity contained by Entity of same type"
salience 65
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
then
$contained.removeFromGraph();
retract($contained);
end
rule "merge intersecting Entities of same type"
salience 64
when
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
then
$first.removeFromGraph();
$second.removeFromGraph();
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
retract($first);
retract($second);
insert(mergedEntity);
end
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then
$entity.removeFromGraph();
retract($entity)
end
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$entity.addEngines($recommendation.getEngines());
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: RedactionEntity(entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of lower rank, when equal boundaries and entityType"
salience 32
when
$higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
$lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
then
$lowerRank.removeFromGraph();
retract($lowerRank);
end
// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
rule "remove duplicate FileAttributes"
salience 64
when
$fileAttribute: FileAttribute($label: label, $value: value)
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
then
retract($duplicate);
end
// --------------------------------------- local dictionary search -------------------------------------------------------------------
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
insert(entity);
});
end
Kilian Schuettler