From c68c6f9ba99b062630244f774929a1da73972803 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 5 May 2023 13:43:00 +0200 Subject: [PATCH] RED-6009 - Document Tree Structure * updated rules to production syngenta rules --- .../graph/entity/RedactionEntity.java | 1 - .../document/graph/nodes/TableNode.java | 19 +- .../services/EntityCreationService.java | 22 +- .../utils/RedactionSearchUtility.java | 38 +- .../v1/server/redaction/model/Section.java | 2 +- .../model/dictionary/Dictionary.java | 25 + .../v1/server/redaction/utils/Patterns.java | 2 +- .../v1/server/RedactionIntegrationV2Test.java | 5 +- .../test/resources/drools/prod_syngenta.drl | 0 .../resources/drools/prod_syngenta_new.drl | 561 ++++++++++++++++++ 10 files changed, 650 insertions(+), 25 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta.drl create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java index db5131ab..05af9bc3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java @@ -123,7 +123,6 @@ public class RedactionEntity { public void addMatchedRule(int ruleNumber) { matchedRules.add(ruleNumber); - engines.add(Engine.RULE); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java index 900fa9ea..83685f14 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java @@ -78,10 +78,10 @@ public class TableNode implements SemanticNode { } - public Stream streamEntitiesWhereRowContainsEntitiesOfType(String type) { + public Stream streamEntitiesWhereRowContainsEntitiesOfType(List types) { List rowsWithEntityOfType = getEntities().stream() - .filter(redactionEntity -> redactionEntity.getType().equals(type)) + .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType()))) .map(RedactionEntity::getIntersectingNodes) .filter(node -> node instanceof TableCellNode) .map(node -> (TableCellNode) node) @@ -143,7 +143,7 @@ public class TableNode implements SemanticNode { public boolean hasHeader(String header) { - return streamHeaders().anyMatch(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().contains(header)); + return streamHeaders().anyMatch(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().strip().equals(header)); } @@ -159,6 +159,19 @@ public class TableNode implements SemanticNode { } + public List getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) { + + return redactionEntity.getIntersectingNodes() + .stream() + .filter(node -> node instanceof TableCellNode) + .map(node -> (TableCellNode) node) + .flatMap(tableCellNode -> streamRow(tableCellNode.getRow())) + .map(cell -> cell.getEntitiesOfType(type)) + .flatMap(Collection::stream) + .toList(); + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index 7a683305..a4ea20a1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -100,7 +100,27 @@ public class EntityCreationService { public Stream byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegex(regexPattern, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); + return byRegex(regexPattern, type, entityType, 0, node); + } + + + public Stream byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, SemanticNode node) { + + return byRegexIgnoreCase(regexPattern, type, entityType, 0, node); + } + + + public Stream byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { + + return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); + } + + + public Stream byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { + + return RedactionSearchUtility.findBoundariesByRegexCaseInsensitive(regexPattern, group, node.buildTextBlock()) + .stream() + .map(boundary -> byBoundary(boundary, type, entityType, node)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java index e0bb8d8c..0383fc36 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java @@ -72,28 +72,36 @@ public class RedactionSearchUtility { } return expandedStart; } - - - public static List findBoundariesByRegex(String regexPattern, CharSequence searchText) { - - Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); - Matcher matcher = pattern.matcher(searchText); - List boundaries = new LinkedList<>(); - while (matcher.find()) { - boundaries.add(new Boundary(matcher.start(), matcher.end())); - } - return boundaries; - - } - + public static List findBoundariesByRegex(String regexPattern, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); + return getBoundariesByPattern(textBlock, 0, pattern); + + } + + + public static List findBoundariesByRegex(String regexPattern, int group, TextBlock textBlock) { + + Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); + return getBoundariesByPattern(textBlock, group, pattern); + } + + + public static List findBoundariesByRegexCaseInsensitive(String regexPattern, int group, TextBlock textBlock) { + + Pattern pattern = Patterns.getCompiledPattern(regexPattern, true); + return getBoundariesByPattern(textBlock, group, pattern); + } + + + private static List getBoundariesByPattern(TextBlock textBlock, int group, Pattern pattern) { + Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary())); List boundaries = new LinkedList<>(); while (matcher.find()) { - boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start())); + boundaries.add(new Boundary(matcher.start(group) + textBlock.getBoundary().start(), matcher.end(group) + textBlock.getBoundary().start())); } return boundaries; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 94796c3a..cc6649cf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1375,7 +1375,7 @@ public class Section { if (addAsRecommendations && !isLocal()) { String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " "; - Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER; + Pattern pattern = Patterns.AUTHOR_TABLE_SPLITTER; Matcher matcher = pattern.matcher(cleanedWord); while (matcher.find()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java index bbdb44fc..b33c1163 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java @@ -6,10 +6,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import lombok.Data; import lombok.Getter; @@ -102,4 +106,25 @@ public class Dictionary { } } + + public void addLocalDictionaryEntry(RedactionEntity redactionEntity) { + + addLocalDictionaryEntry(redactionEntity.getType(), redactionEntity.getValue(), false); + } + + + public void addMultipleAuthorsAsRecommendation(RedactionEntity redactionEntity) { + + String cleanedWord = redactionEntity.getValue().replaceAll(",", " ").replaceAll(" ", " ").trim() + " "; + Pattern pattern = Patterns.AUTHOR_TABLE_SPLITTER; + Matcher matcher = pattern.matcher(cleanedWord); + + while (matcher.find()) { + String match = matcher.group().trim(); + if (match.length() >= 3) { + addLocalDictionaryEntry(redactionEntity.getType(), match, true); + } + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java index c967a4ba..b1828e88 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java @@ -11,7 +11,7 @@ public final class Patterns { public static Map patternCache = new HashMap<>(); - public static Pattern AUTHOR_TABLE_SPITTER = Pattern.compile( + public static Pattern AUTHOR_TABLE_SPLITTER = Pattern.compile( "(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2}\\.){1,3})|(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2} ){1,3})"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java index 0848b794..352fc2cf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java @@ -131,9 +131,8 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLogEntry.isExcluded()).isEqualTo(false); assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); - assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2); + assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); - } @@ -170,7 +169,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest assertThat(redactionLogEntry.isExcluded()).isEqualTo(false); assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true); - assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2); + assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1); assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta.drl new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl new file mode 100644 index 00000000..41ff4f74 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl @@ -0,0 +1,561 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch; +import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType; + +import java.util.List; +import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt; +import java.util.LinkedList; +import java.util.HashSet; + +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import java.util.Set +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility; + +global DocumentGraph document +global EntityCreationService entityCreationService +global ManualRedactionApplicationService manualRedactionApplicationService +global NerEntitiesAdapter nerEntitiesAdapter +global Dictionary dictionary + +// --------------------------------------- queries ------------------------------------------------------------------- + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +// --------------------------------------- Syngenta specific laboratory recommendation ------------------------------------------------------------------- + +rule "0: Recommend CTL/BL laboratory that start with BL or CTL" + when + $section: SectionNode(containsString("CT") || containsString("BL")) + then + /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */ + entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(entity -> { + entity.addMatchedRule(0); + entity.addEngine(Engine.RULE); + insert(entity); + }); + end +// --------------------------------------- CBI Rules ------------------------------------------------------------------- + +rule "1: Redact CBI Authors" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY) + then + $entity.setRedaction(true); + $entity.addMatchedRule(1); + $entity.setRedactionReason("Author found"); + $entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "2: Don't redact CBI Address (Non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY) + then + $entity.setRedaction(false); + $entity.addMatchedRule(2); + $entity.setRedactionReason("Address found for non vertebrate study"); + end + +rule "3: Redact CBI Address (Vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY) + then + $entity.setRedaction(true); + $entity.addMatchedRule(4); + $entity.setRedactionReason("Address found"); + $entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" + when + $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction) + then + RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); + falsePositive.addMatchedRule(5); + insert(falsePositive); + end + +rule "6: Redact all Cell's with Header Author(s) as CBI_author" + when + $table: TableNode(hasHeader("Author(s)")) + then + $table.streamTableCellsWithHeader("Author(s)") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.addMatchedRule(6); + redactionEntity.addEngine(Engine.RULE); + redactionEntity.setRedactionReason("Author(s) found"); + redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); + insert(redactionEntity); + }); + end + +rule "8: Redact all Cell's with Header Author(s) as CBI_author" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $table: TableNode(hasHeader("Author")) + then + $table.streamTableCellsWithHeader("Author") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(true); + redactionEntity.addMatchedRule(8); + redactionEntity.addEngine(Engine.RULE); + redactionEntity.setRedactionReason("Author found"); + redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); + insert(redactionEntity); + }); + end + +rule "10: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $table: TableNode(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) + then + $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); + end + +rule "14: Add CBI_author with \"et al.\" Regex" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + $section: SectionNode(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section) + .forEach(entity -> { + entity.setRedaction(true); + entity.setRedactionReason("Author found by \"et al\" regex"); + entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); + entity.addMatchedRule(14); + entity.addEngine(Engine.RULE); + insert(entity); + dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); + }); + end + +rule "16: Add recommendation for Addresses in Test Organism sections" + when + $section: SectionNode(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) + then + entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(entity -> { + entity.setRedactionReason("Line after \"Source\" in Test Organism Section"); + entity.addEngine(Engine.RULE); + entity.addMatchedRule(16); + insert(entity); + }); + end + +rule "17: Add recommendation for Addresses in Test Animals sections" + when + $section: SectionNode(excludesTables, containsString("Species:"), containsString("Source:")) + then + entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(entity -> { + entity.setRedactionReason("Line after \"Source:\" in Test Animals Section"); + entity.addEngine(Engine.RULE); + entity.addMatchedRule(17); + insert(entity); + }); + end + +rule "18: Do not redact Names and Addresses if published information found in section without tables" + when + $section: SectionNode(excludesTables && + hasEntitiesOfType("published_information"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + List publishedInformationEntities = $section.getEntitiesOfType("published_information"); + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(false); + redactionEntity.setRedactionReason("Published Information found"); + redactionEntity.addReferences(publishedInformationEntities); + }); + end + +rule "18.0: Do not redact Names and Addresses if published information found in same table row" + when + $table: TableNode(hasEntitiesOfType("published_information"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.setRedaction(false); + redactionEntity.setRedactionReason("Published Information found in row"); + redactionEntity.addReferences($table.getEntitiesOfTypeInSameRow("published_information", redactionEntity)); + }); + end + +// --------------------------------------- PII rules ------------------------------------------------------------------- + +rule "19: Redact all PII" + when + $pii: RedactionEntity(type == "PII", redaction == false) + then + $pii.setRedaction(true); + $pii.setRedactionReason("Personal Information found"); + $pii.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.addMatchedRule(19); + end + +rule "21: Redact Emails by RegEx (Non vertebrate study)" + when + $section: SectionNode(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> { + emailEntity.setRedaction(true); + emailEntity.setRedactionReason("Found by Email Regex"); + emailEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + emailEntity.addMatchedRule(21); + insert(emailEntity); + }); + end + + +rule "25: Redact Phone and Fax by RegEx" + when + $section: SectionNode(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Ph.") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) + then + entityCreationService.byRegex("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) + .forEach(contactEntity -> { + contactEntity.setRedaction(true); + contactEntity.setRedactionReason("Found by Email Regex"); + contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + contactEntity.addMatchedRule(26); + insert(contactEntity); + }); + end + + +rule "27: Redact AUTHOR(S)" + when + $section: SectionNode(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.addMatchedRule(27); + authorEntity.setRedactionReason("AUTHOR(S) was found"); + authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + insert(authorEntity); + }); + end + + +rule "29: Redact AUTHOR(S)" + when + $section: SectionNode(excludesTables, containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.setRedaction(true); + authorEntity.addMatchedRule(29); + authorEntity.setRedactionReason("AUTHOR(S) was found"); + authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); + insert(authorEntity); + }); + end + +rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value == "Yes") + $section: SectionNode(excludesTables, containsString("PERFORMING LABORATORY:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.setRedaction(false); + laboratoryEntity.addMatchedRule(31); + laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study"); + dictionary.addLocalDictionaryEntry(laboratoryEntity); + insert(laboratoryEntity); + }); + end + +rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value == "Yes") + $section: SectionNode(excludesTables, containsString("PERFORMING LABORATORY:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.setRedaction(true); + laboratoryEntity.addMatchedRule(32); + laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found"); + laboratoryEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); + dictionary.addLocalDictionaryEntry(laboratoryEntity); + insert(laboratoryEntity); + }); + end + +// --------------------------------------- other rules ------------------------------------------------------------------- + +rule "33: Purity Hint" + when + $section: SectionNode(containsStringIgnoreCase("purity")) + then + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) + .forEach(hint -> { + hint.addEngine(Engine.RULE); + hint.addMatchedRule(33); + }); + end + +rule "34: Redact signatures (not Vertebrate Study)" + when + not FileAttribute(label == "Vertebrate Study", value == "Yes") + $signature: ImageNode(imageType == ImageType.SIGNATURE) + then + $signature.setRedaction(true); + $signature.setMatchedRule(34); + $signature.setRedactionReason("Signature Found"); + $signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "35: Redact signatures (Vertebrate Study)" + when + FileAttribute(label == "Vertebrate Study", value == "Yes") + $signature: ImageNode(imageType == ImageType.SIGNATURE) + then + $signature.setRedaction(true); + $signature.setMatchedRule(35); + $signature.setRedactionReason("Signature Found"); + $signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + + +rule "36: Redact logos" + when + $logo: ImageNode(imageType == ImageType.LOGO) + then + $logo.setRedaction(true); + $logo.setMatchedRule(36); + $logo.setRedactionReason("Logo Found"); + $logo.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +// --------------------------------------- NER Entities rules ------------------------------------------------------------------- + +rule "add NER Entities of type CBI_author" + salience 999 + when + nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) + then + nerEntities.streamEntitiesOfType("CBI_author") + .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) + .forEach(entity -> insert(entity)); + end + +rule "combine and add NER Entities as CBI_address" + salience 999 + when + nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY")) + then + nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) + .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) + .forEach(entity -> { + entity.addEngine(Engine.NER); + insert(entity); + }); + end + +// --------------------------------------- manual redaction rules ------------------------------------------------------------------- + +rule "Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId) + $entityToBeResized: RedactionEntity(matchesAnnotationId($id)) + then + manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($entityToBeResized); + end + +rule "Apply id removals that are valid and not in forced redactions to Entity" + salience 128 + when + IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) + then + $entityToBeRemoved.removeFromGraph(); + retract($entityToBeRemoved); + end + +rule "Apply id removals that are valid and not in forced redactions to Image" + salience 128 + when + IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $entityToBeRemoved: ImageNode($id == id) + then + $entityToBeRemoved.setIgnored(true); + end + +rule "Apply force redaction" + salience 128 + when + ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $entityToForce: RedactionEntity(matchesAnnotationId($id)) + then + $entityToForce.setLegalBasis($legalBasis); + $entityToForce.setRedaction(true); + $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + end + +rule "Apply image recategorization" + salience 128 + when + ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) + $image: ImageNode($id == id) + then + $image.setImageType(parseImageType($imageType)); + end + +// --------------------------------------- merging rules ------------------------------------------------------------------- + +rule "remove Entity contained by Entity of same type" + salience 65 + when + $larger: RedactionEntity($type: type, $entityType: entityType) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + then + $contained.removeFromGraph(); + retract($contained); + end + +rule "merge intersecting Entities of same type" + salience 64 + when + $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) + $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) + then + $first.removeFromGraph(); + $second.removeFromGraph(); + RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + retract($first); + retract($second); + insert(mergedEntity); + end + +rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 64 + when + $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) + $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + then + $entity.removeFromGraph(); + retract($entity) + end + +rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 64 + when + $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) + $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" + salience 256 + when + $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $entity.addEngines($recommendation.getEngines()); + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 256 + when + $entity: RedactionEntity(entityType == EntityType.ENTITY) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of lower rank, when equal boundaries and entityType" + salience 32 + when + $higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary) + $lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction) + then + $lowerRank.removeFromGraph(); + retract($lowerRank); + end + +// --------------------------------------- FileAttribute Rules ------------------------------------------------------------------- + +rule "remove duplicate FileAttributes" + salience 64 + when + $fileAttribute: FileAttribute($label: label, $value: value) + $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value) + then + retract($duplicate); + end + +// --------------------------------------- local dictionary search ------------------------------------------------------------------- + +rule "run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) + .forEach(entity -> { + entity.addEngine(Engine.RULE); + insert(entity); + }); + end