7adb6508e3
* changed entityCreationService such that every entity can't be created twice in order to avoid endless loops * made analysis service more verbose * made dictionary service less verbose
773 lines
34 KiB
Plaintext
773 lines
34 KiB
Plaintext
package drools
|
||
|
||
import static java.lang.String.format;
|
||
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
|
||
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
|
||
|
||
import java.util.List;
|
||
import java.util.LinkedList;
|
||
import java.util.Set;
|
||
import java.util.stream.Collectors;
|
||
import java.util.Collection;
|
||
import java.util.stream.Stream;
|
||
import java.util.Optional;
|
||
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
|
||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
|
||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
|
||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
|
||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
|
||
|
||
global Document document
|
||
global EntityCreationService entityCreationService
|
||
global ManualRedactionApplicationService manualRedactionApplicationService
|
||
global NerEntitiesAdapter nerEntitiesAdapter
|
||
global Dictionary dictionary
|
||
|
||
//------------------------------------ queries ------------------------------------
|
||
|
||
query "getFileAttributes"
|
||
$fileAttribute: FileAttribute()
|
||
end
|
||
|
||
//------------------------------------ Syngenta specific rules ------------------------------------
|
||
|
||
// Rule unit: SYN.1
|
||
rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
|
||
when
|
||
$section: Section(containsString("CT") || containsString("BL"))
|
||
then
|
||
/* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
|
||
entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
|
||
.forEach(entity -> {
|
||
entity.skip("SYN.1.0", "");
|
||
entity.addEngine(Engine.RULE);
|
||
insert(entity);
|
||
});
|
||
end
|
||
|
||
|
||
//------------------------------------ CBI rules ------------------------------------
|
||
|
||
// Rule unit: CBI.0
|
||
rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$entity: RedactionEntity(type == "CBI_author", dictionaryEntry)
|
||
then
|
||
$entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$entity: RedactionEntity(type == "CBI_author", dictionaryEntry)
|
||
then
|
||
$entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.1
|
||
rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$entity: RedactionEntity(type == "CBI_address", dictionaryEntry)
|
||
then
|
||
$entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
|
||
end
|
||
|
||
rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$entity: RedactionEntity(type == "CBI_address", dictionaryEntry)
|
||
then
|
||
$entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.2
|
||
rule "CBI.2.0: Don't redact genitive CBI_author"
|
||
when
|
||
$entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied())
|
||
then
|
||
entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document)
|
||
.ifPresent(falsePositive -> {
|
||
falsePositive.skip("CBI.2.0", "Genitive Author found");
|
||
insert(falsePositive);
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.7
|
||
rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables"
|
||
when
|
||
$section: Section(!hasTables(),
|
||
hasEntitiesOfType("published_information"),
|
||
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
||
then
|
||
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
||
.forEach(redactionEntity -> {
|
||
redactionEntity.skipWithReferences(
|
||
"CBI.7.0",
|
||
"Published Information found in section",
|
||
$section.getEntitiesOfType("published_information")
|
||
);
|
||
});
|
||
end
|
||
|
||
rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
|
||
when
|
||
$table: Table(hasEntitiesOfType("published_information"),
|
||
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
||
then
|
||
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
||
.forEach(redactionEntity -> {
|
||
redactionEntity.skipWithReferences(
|
||
"CBI.7.1",
|
||
"Published Information found in row",
|
||
$table.getEntitiesOfTypeInSameRow("published_information", redactionEntity)
|
||
);
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.9
|
||
rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$table: Table(hasHeader("Author(s)"))
|
||
then
|
||
$table.streamTableCellsWithHeader("Author(s)")
|
||
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
||
.filter(Optional::isPresent)
|
||
.map(Optional::get)
|
||
.forEach(redactionEntity -> {
|
||
redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
redactionEntity.addEngine(Engine.RULE);
|
||
insert(redactionEntity);
|
||
});
|
||
end
|
||
|
||
rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$table: Table(hasHeader("Author"))
|
||
then
|
||
$table.streamTableCellsWithHeader("Author")
|
||
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
||
.filter(Optional::isPresent)
|
||
.map(Optional::get)
|
||
.forEach(redactionEntity -> {
|
||
redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
redactionEntity.addEngine(Engine.RULE);
|
||
insert(redactionEntity);
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.10
|
||
rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$table: Table(hasHeader("Author(s)"))
|
||
then
|
||
$table.streamTableCellsWithHeader("Author(s)")
|
||
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
||
.filter(Optional::isPresent)
|
||
.map(Optional::get)
|
||
.forEach(redactionEntity -> {
|
||
redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
redactionEntity.addEngine(Engine.RULE);
|
||
insert(redactionEntity);
|
||
});
|
||
end
|
||
|
||
rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$table: Table(hasHeader("Author"))
|
||
then
|
||
$table.streamTableCellsWithHeader("Author")
|
||
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
||
.filter(Optional::isPresent)
|
||
.map(Optional::get)
|
||
.forEach(redactionEntity -> {
|
||
redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
redactionEntity.addEngine(Engine.RULE);
|
||
insert(redactionEntity);
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.11
|
||
rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
salience -1
|
||
when
|
||
$table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
|
||
then
|
||
$table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.16
|
||
rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(containsString("et al."))
|
||
then
|
||
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
|
||
.forEach(entity -> {
|
||
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
entity.addEngine(Engine.RULE);
|
||
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
|
||
insert(entity);
|
||
});
|
||
end
|
||
|
||
rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(containsString("et al."))
|
||
then
|
||
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
|
||
.forEach(entity -> {
|
||
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
entity.addEngine(Engine.RULE);
|
||
insert(entity);
|
||
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.17
|
||
rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
|
||
when
|
||
$section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
|
||
then
|
||
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
|
||
.forEach(entity -> {
|
||
entity.addEngine(Engine.RULE);
|
||
entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section");
|
||
insert(entity);
|
||
});
|
||
end
|
||
|
||
rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
|
||
when
|
||
$section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
|
||
then
|
||
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
|
||
.forEach(entity -> {
|
||
entity.addEngine(Engine.RULE);
|
||
entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section");
|
||
insert(entity);
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: CBI.20
|
||
rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value == "Yes")
|
||
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
|
||
then
|
||
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
|
||
.forEach(laboratoryEntity -> {
|
||
laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
|
||
laboratoryEntity.addEngine(Engine.RULE);
|
||
dictionary.addLocalDictionaryEntry(laboratoryEntity);
|
||
insert(laboratoryEntity);
|
||
});
|
||
end
|
||
|
||
rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value == "Yes")
|
||
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
|
||
then
|
||
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
|
||
.forEach(laboratoryEntity -> {
|
||
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
laboratoryEntity.addEngine(Engine.RULE);
|
||
dictionary.addLocalDictionaryEntry(laboratoryEntity);
|
||
insert(laboratoryEntity);
|
||
});
|
||
end
|
||
|
||
|
||
//------------------------------------ PII rules ------------------------------------
|
||
|
||
// Rule unit: PII.0
|
||
rule "PII.0.0: Redact all PII (non vertebrate study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$pii: RedactionEntity(type == "PII", dictionaryEntry)
|
||
then
|
||
$pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
rule "PII.0.1: Redact all PII (vertebrate study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$pii: RedactionEntity(type == "PII", dictionaryEntry)
|
||
then
|
||
$pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
|
||
// Rule unit: PII.1
|
||
rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(containsString("@"))
|
||
then
|
||
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
|
||
.forEach(emailEntity -> {
|
||
emailEntity.addEngine(Engine.RULE);
|
||
emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
insert(emailEntity);
|
||
});
|
||
end
|
||
|
||
rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(containsString("@"))
|
||
then
|
||
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
|
||
.forEach(emailEntity -> {
|
||
emailEntity.addEngine(Engine.RULE);
|
||
emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
insert(emailEntity);
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: PII.2
|
||
rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(containsString("Contact") ||
|
||
containsString("Telephone") ||
|
||
containsString("Phone") ||
|
||
containsString("Ph.") ||
|
||
containsString("Fax") ||
|
||
containsString("Tel") ||
|
||
containsString("Ter") ||
|
||
containsString("Mobile") ||
|
||
containsString("Fel") ||
|
||
containsString("Fer"))
|
||
then
|
||
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
|
||
.forEach(contactEntity -> {
|
||
contactEntity.addEngine(Engine.RULE);
|
||
contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
insert(contactEntity);
|
||
});
|
||
end
|
||
|
||
rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(containsString("Contact") ||
|
||
containsString("Telephone") ||
|
||
containsString("Phone") ||
|
||
containsString("Ph.") ||
|
||
containsString("Fax") ||
|
||
containsString("Tel") ||
|
||
containsString("Ter") ||
|
||
containsString("Mobile") ||
|
||
containsString("Fel") ||
|
||
containsString("Fer"))
|
||
then
|
||
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
|
||
.forEach(contactEntity -> {
|
||
contactEntity.addEngine(Engine.RULE);
|
||
contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
insert(contactEntity);
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: PII.9
|
||
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
|
||
then
|
||
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
|
||
.forEach(authorEntity -> {
|
||
authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
authorEntity.addEngine(Engine.RULE);
|
||
insert(authorEntity);
|
||
});
|
||
end
|
||
|
||
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
|
||
then
|
||
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
|
||
.forEach(authorEntity -> {
|
||
authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
authorEntity.addEngine(Engine.RULE);
|
||
insert(authorEntity);
|
||
});
|
||
end
|
||
|
||
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
|
||
then
|
||
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
|
||
.forEach(authorEntity -> {
|
||
authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
authorEntity.addEngine(Engine.RULE);
|
||
insert(authorEntity);
|
||
});
|
||
end
|
||
|
||
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
|
||
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
|
||
then
|
||
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
|
||
.forEach(authorEntity -> {
|
||
authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
authorEntity.addEngine(Engine.RULE);
|
||
insert(authorEntity);
|
||
});
|
||
end
|
||
|
||
|
||
//------------------------------------ Other rules ------------------------------------
|
||
|
||
// Rule unit: ETC.0
|
||
rule "ETC.0.0: Purity Hint"
|
||
when
|
||
$section: Section(containsStringIgnoreCase("purity"))
|
||
then
|
||
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
|
||
.forEach(hint -> {
|
||
hint.addEngine(Engine.RULE);
|
||
hint.skip("ETC.0.0", "");
|
||
});
|
||
end
|
||
|
||
|
||
// Rule unit: ETC.2
|
||
rule "ETC.2.0: Redact signatures (non vertebrate study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value == "Yes")
|
||
$signature: Image(imageType == ImageType.SIGNATURE)
|
||
then
|
||
$signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
rule "ETC.2.0: Redact signatures (vertebrate study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value == "Yes")
|
||
$signature: Image(imageType == ImageType.SIGNATURE)
|
||
then
|
||
$signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
|
||
// Rule unit: ETC.3
|
||
rule "ETC.3.0: Redact logos (vertebrate study)"
|
||
when
|
||
not FileAttribute(label == "Vertebrate Study", value == "Yes")
|
||
$logo: Image(imageType == ImageType.LOGO)
|
||
then
|
||
$logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
rule "ETC.3.1: Redact logos (non vertebrate study)"
|
||
when
|
||
FileAttribute(label == "Vertebrate Study", value == "Yes")
|
||
$logo: Image(imageType == ImageType.LOGO)
|
||
then
|
||
$logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||
end
|
||
|
||
|
||
// Rule unit: ETC.5
|
||
rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
|
||
when
|
||
not FileAttribute(label == "Confidentiality", value == "confidential")
|
||
$dossierRedaction: RedactionEntity(type == "dossier_redaction")
|
||
then
|
||
$dossierRedaction.setIgnored(true);
|
||
update($dossierRedaction);
|
||
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
|
||
end
|
||
|
||
|
||
//------------------------------------ AI rules ------------------------------------
|
||
|
||
// Rule unit: AI.0
|
||
rule "AI.0.0: add all NER Entities of type CBI_author"
|
||
salience 999
|
||
when
|
||
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
|
||
then
|
||
nerEntities.streamEntitiesOfType("CBI_author")
|
||
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
|
||
.filter(Optional::isPresent)
|
||
.map(Optional::get)
|
||
.forEach(entity -> insert(entity));
|
||
end
|
||
|
||
|
||
// Rule unit: AI.1
|
||
rule "AI.1.0: combine and add NER Entities as CBI_address"
|
||
salience 999
|
||
when
|
||
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
|
||
then
|
||
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
|
||
.map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
|
||
.filter(Optional::isPresent)
|
||
.map(Optional::get)
|
||
.forEach(entity -> {
|
||
entity.addEngine(Engine.NER);
|
||
insert(entity);
|
||
});
|
||
end
|
||
|
||
|
||
//------------------------------------ Manual redaction rules ------------------------------------
|
||
|
||
// Rule unit: MAN.0
|
||
rule "MAN.0.0: Apply manual resize redaction"
|
||
salience 128
|
||
when
|
||
$resizeRedaction: ManualResizeRedaction($id: annotationId)
|
||
$entityToBeResized: RedactionEntity(matchesAnnotationId($id))
|
||
then
|
||
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
|
||
retract($resizeRedaction);
|
||
update($entityToBeResized);
|
||
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
|
||
end
|
||
|
||
|
||
// Rule unit: MAN.1
|
||
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
|
||
salience 128
|
||
when
|
||
$idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
|
||
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
|
||
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
|
||
then
|
||
$entityToBeRemoved.setIgnored(true);
|
||
update($entityToBeRemoved);
|
||
retract($idRemoval);
|
||
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
|
||
end
|
||
|
||
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
|
||
salience 128
|
||
when
|
||
$idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
|
||
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
|
||
$imageEntityToBeRemoved: Image($id == id)
|
||
then
|
||
$imageEntityToBeRemoved.setIgnored(true);
|
||
update($imageEntityToBeRemoved);
|
||
retract($idRemoval);
|
||
update($imageEntityToBeRemoved.getParent());
|
||
end
|
||
|
||
|
||
// Rule unit: MAN.2
|
||
rule "MAN.2.0: Apply force redaction"
|
||
salience 128
|
||
when
|
||
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
|
||
$entityToForce: RedactionEntity(matchesAnnotationId($id))
|
||
then
|
||
$entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
|
||
$entityToForce.setRemoved(false);
|
||
$entityToForce.setIgnored(false);
|
||
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
|
||
update($entityToForce);
|
||
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
|
||
retract($force);
|
||
end
|
||
|
||
|
||
// Rule unit: MAN.3
|
||
rule "MAN.3.0: Apply image recategorization"
|
||
salience 128
|
||
when
|
||
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
|
||
$imageToBeRecategorized: Image($id == id)
|
||
then
|
||
$imageToBeRecategorized.setImageType(ImageType.fromString($imageType));
|
||
update($imageToBeRecategorized);
|
||
retract($recategorization);
|
||
update($imageToBeRecategorized.getParent());
|
||
end
|
||
|
||
|
||
//------------------------------------ Entity merging rules ------------------------------------
|
||
|
||
// Rule unit: X.0
|
||
rule "X.0.0: remove Entity contained by Entity of same type"
|
||
salience 65
|
||
when
|
||
$larger: RedactionEntity($type: type, $entityType: entityType)
|
||
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$contained.remove();
|
||
retract($contained);
|
||
end
|
||
|
||
|
||
// Rule unit: X.1
|
||
rule "X.1.0: merge intersecting Entities of same type"
|
||
salience 64
|
||
when
|
||
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$first.remove();
|
||
$second.remove();
|
||
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
|
||
retract($first);
|
||
retract($second);
|
||
insert(mergedEntity);
|
||
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
|
||
end
|
||
|
||
|
||
// Rule unit: X.2
|
||
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
||
salience 64
|
||
when
|
||
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
|
||
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$entity.getIntersectingNodes().forEach(node -> update(node));
|
||
$entity.remove();
|
||
retract($entity)
|
||
end
|
||
|
||
|
||
// Rule unit: X.3
|
||
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
|
||
salience 64
|
||
when
|
||
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
|
||
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$recommendation.remove();
|
||
retract($recommendation);
|
||
end
|
||
|
||
|
||
// Rule unit: X.4
|
||
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
|
||
salience 256
|
||
when
|
||
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
|
||
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$entity.addEngines($recommendation.getEngines());
|
||
$recommendation.remove();
|
||
retract($recommendation);
|
||
end
|
||
|
||
|
||
// Rule unit: X.5
|
||
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
|
||
salience 256
|
||
when
|
||
$entity: RedactionEntity(entityType == EntityType.ENTITY)
|
||
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$recommendation.remove();
|
||
retract($recommendation);
|
||
end
|
||
|
||
|
||
// Rule unit: X.6
|
||
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
|
||
salience 32
|
||
when
|
||
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)
|
||
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
|
||
then
|
||
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
|
||
$lowerRank.remove();
|
||
retract($lowerRank);
|
||
end
|
||
|
||
|
||
//------------------------------------ File attributes rules ------------------------------------
|
||
|
||
// Rule unit: FA.1
|
||
rule "FA.1.0: remove duplicate FileAttributes"
|
||
salience 64
|
||
when
|
||
$fileAttribute: FileAttribute($label: label, $value: value)
|
||
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
|
||
then
|
||
retract($duplicate);
|
||
end
|
||
|
||
|
||
//------------------------------------ Local dictionary search rules ------------------------------------
|
||
|
||
// Rule unit: LDS.0
|
||
rule "LDS.0.0: run local dictionary search"
|
||
agenda-group "LOCAL_DICTIONARY_ADDS"
|
||
salience -999
|
||
when
|
||
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
|
||
then
|
||
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
|
||
.forEach(entity -> {
|
||
entity.addEngine(Engine.RULE);
|
||
insert(entity);
|
||
});
|
||
end
|