59827eab81
* refactored DocumentData to 4 different jsons to make reading only text possible * wip integration of new rules into workflow
391 lines
17 KiB
Plaintext
391 lines
17 KiB
Plaintext
package drools
|
||
|
||
import static java.lang.String.format;
|
||
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
|
||
import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
|
||
|
||
|
||
import java.util.List;
|
||
import java.util.LinkedList;
|
||
import java.util.HashSet;
|
||
|
||
import com.iqser.red.service.redaction.v1.server.document.graph.*
|
||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
|
||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
|
||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*
|
||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||
import java.util.Set
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
|
||
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
|
||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||
|
||
global DocumentGraph document
|
||
global EntityCreationService entityCreationService
|
||
global Dictionary dictionary
|
||
|
||
|
||
query "getFileAttributes"
|
||
$fileAttribute: FileAttribute()
|
||
end
|
||
|
||
rule "merge intersecting Entities of same type"
|
||
salience 100
|
||
|
||
when
|
||
$first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
$second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$first.removeFromGraph();
|
||
$second.removeFromGraph();
|
||
EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
|
||
insert(mergedEntity);
|
||
retract($first);
|
||
retract($second);
|
||
end
|
||
|
||
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
||
salience 100
|
||
|
||
when
|
||
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
|
||
entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
entity.removeFromGraph();
|
||
retract(entity);
|
||
end
|
||
|
||
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
|
||
salience 100
|
||
|
||
when
|
||
$falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
|
||
$recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$recommendation.removeFromGraph();
|
||
retract($recommendation);
|
||
end
|
||
|
||
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
|
||
salience 100
|
||
|
||
when
|
||
$entity: EntityNode($type: type, entityType == EntityType.ENTITY)
|
||
$recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$recommendation.removeFromGraph();
|
||
retract($recommendation);
|
||
end
|
||
|
||
rule "remove Entity of lower rank, when equal boundaries and entityType"
|
||
salience 100
|
||
|
||
when
|
||
$higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
|
||
$lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
|
||
then
|
||
$lowerRank.removeFromGraph();
|
||
retract($lowerRank);
|
||
end
|
||
|
||
rule "run local dictionary search"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
salience -999
|
||
|
||
when
|
||
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
|
||
then
|
||
Set<EntityNode> entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
|
||
System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
|
||
entityNodes.forEach(entityNode -> insert(entityNode));
|
||
end
|
||
|
||
|
||
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
||
|
||
rule "1: Redact CBI Authors (Non vertebrate study)"
|
||
no-loop true
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
|
||
then
|
||
$entity.setRedaction(true);
|
||
setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
|
||
update($entity)
|
||
end
|
||
|
||
rule "2: Redact CBI Authors (Vertebrate study)"
|
||
no-loop true
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
|
||
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
|
||
then
|
||
$entity.setRedaction(true);
|
||
setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
|
||
update($entity)
|
||
end
|
||
|
||
rule "3: Don't redact CBI Address (Non vertebrate study)"
|
||
no-loop true
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
|
||
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
|
||
then
|
||
setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
|
||
update($entity)
|
||
end
|
||
|
||
rule "4: Redact CBI Address (Vertebrate study)"
|
||
no-loop true
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
|
||
then
|
||
$entity.setRedaction(true);
|
||
setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
|
||
update($entity)
|
||
end
|
||
|
||
rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
|
||
|
||
when
|
||
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
|
||
then
|
||
EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
|
||
setFields($entity, 5, "Genitive Author", null, Engine.RULE);
|
||
insert(entity);
|
||
end
|
||
|
||
|
||
rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
|
||
when
|
||
authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
|
||
then
|
||
EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
|
||
setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
|
||
insert(entity);
|
||
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
|
||
end
|
||
|
||
rule "7: Add CBI_author with \"et al.\" Regex"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
|
||
when
|
||
$section: SectionNode(containsString("et al."))
|
||
then
|
||
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
|
||
setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
|
||
end
|
||
|
||
rule "8: Add recommendation for Addresses in Test Organism sections"
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||
$section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
|
||
then
|
||
Set<EntityNode> entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
|
||
setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
end
|
||
|
||
|
||
rule "9: Add recommendation for Addresses in Test Animals sections"
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||
$section: SectionNode(containsString("Species:") && containsString("Source:"))
|
||
then
|
||
Set<EntityNode> entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
|
||
setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
end
|
||
|
||
rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
|
||
|
||
when
|
||
$section : SectionNode(containsString("BL") || containsString("CT"))
|
||
then
|
||
Set<EntityNode> entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section);
|
||
setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
|
||
end
|
||
|
||
|
||
// --------------------------------------- PII rules -------------------------------------------------------------------
|
||
|
||
|
||
rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
|
||
no-loop true
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
|
||
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
|
||
then
|
||
$entity.setRedaction(true);
|
||
setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
|
||
update($entity)
|
||
end
|
||
|
||
|
||
|
||
rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
|
||
no-loop true
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
|
||
then
|
||
$entity.setRedaction(true);
|
||
setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
|
||
update($entity)
|
||
end
|
||
|
||
rule "12: Redact Emails by RegEx (Non vertebrate study)"
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
|
||
$section: SectionNode(containsString("@"))
|
||
then
|
||
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
|
||
setFields(entities, 12, "Found by email regex", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
end
|
||
|
||
rule "13: Redact Emails by RegEx (Vertebrate study)"
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||
$section: SectionNode(containsString("@"))
|
||
then
|
||
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
|
||
setFields(entities, 13, "Found by email regex", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
end
|
||
|
||
|
||
rule "14: Redact line after contact information (Non vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
|
||
$string: String() from List.of("Contact point:",
|
||
"Contact:",
|
||
"Alternative contact:",
|
||
"European contact:",
|
||
"No:",
|
||
"Contact:",
|
||
"Tel.:",
|
||
"Tel:",
|
||
"Telephone number:",
|
||
"Telephone No:",
|
||
"Telephone:",
|
||
"Phone No:",
|
||
"Phone:",
|
||
"Fax number:",
|
||
"Fax:",
|
||
"E-mail:",
|
||
"Email:",
|
||
"e-mail:",
|
||
"E-mail address:")
|
||
$section: SectionNode(containsString($string))
|
||
then
|
||
|
||
Set<EntityNode> entities = new HashSet<>();
|
||
entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section));
|
||
setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
|
||
end
|
||
|
||
|
||
rule "15: Redact line after contact information (Vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||
$string: String() from List.of("Contact point:",
|
||
"Contact:",
|
||
"Alternative contact:",
|
||
"European contact:",
|
||
"No:",
|
||
"Contact:",
|
||
"Tel.:",
|
||
"Tel:",
|
||
"Telephone number:",
|
||
"Telephone No:",
|
||
"Telephone:",
|
||
"Phone No:",
|
||
"Phone:",
|
||
"Fax number:",
|
||
"Fax:",
|
||
"E-mail:",
|
||
"Email:",
|
||
"e-mail:",
|
||
"E-mail address:")
|
||
$section: SectionNode(containsString($string))
|
||
then
|
||
Set<EntityNode> entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section);
|
||
setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
|
||
end
|
||
|
||
|
||
rule "16: redact line between contact keywords"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
|
||
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
|
||
then
|
||
Set<EntityNode> entities = new HashSet<>();
|
||
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
|
||
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
|
||
setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
|
||
end
|
||
|
||
rule "17: redact line between contact keywords"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
|
||
when
|
||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
|
||
then
|
||
Set<EntityNode> entities = new HashSet<>();
|
||
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
|
||
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
|
||
setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
|
||
end
|
||
|
||
rule "18: Redact Phone and Fax by RegEx"
|
||
|
||
when
|
||
$section: SectionNode(containsString("Contact") ||
|
||
containsString("Telephone") ||
|
||
containsString("Phone") ||
|
||
containsString("Fax") ||
|
||
containsString("Tel") ||
|
||
containsString("Ter") ||
|
||
containsString("Mobile") ||
|
||
containsString("Fel") ||
|
||
containsString("Fer"))
|
||
then
|
||
Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
|
||
setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
|
||
entities.forEach(entity -> insert(entity));
|
||
end
|