From 9d7180923df0360ee342e71b613da5a546fe4afa Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 6 Jul 2023 17:14:29 +0200 Subject: [PATCH] RED-6929: fix acceptance rules/tests --- .../adapter/RedactionLogEntryAdapter.java | 21 +- .../document/graph/entity/MatchedRule.java | 6 + .../graph/entity/MatchedRuleHolder.java | 48 +- .../graph/entity/RedactionEntity.java | 18 - .../document/graph/nodes/Image.java | 18 - .../document/graph/nodes/SemanticNode.java | 97 +- .../services/EntityCreationService.java | 55 +- .../ManualRedactionApplicationService.java | 15 +- .../redaction/service/AnalyzeService.java | 14 +- .../service/DroolsExecutionService.java | 7 +- .../service/EntityRedactionService.java | 32 +- ...ManualRedactionSurroundingTextService.java | 4 +- .../service/RedactionChangeLogService.java | 2 +- .../v1/server/DocumineFloraTest.java | 5 +- .../v1/server/RedactionAcceptanceTest.java | 88 +- .../v1/server/RedactionIntegrationTest.java | 5 +- ...ocumentEntityInsertionIntegrationTest.java | 23 +- .../DocumentPerformanceIntegrationTest.java | 6 +- .../ManualResizeRedactionIntegrationTest.java | 23 +- .../adapter/NerEntitiesAdapterTest.java | 10 + .../resources/drools/acceptance_rules.drl | 158 +--- .../src/test/resources/drools/all_rules.drl | 296 ++---- .../drools/manual_redaction_rules.drl | 58 +- .../src/test/resources/drools/rules.drl | 247 ++--- .../src/test/resources/drools/rules_v2.drl | 47 +- .../src/test/resources/log4j2-test.xml | 2 +- .../EFSA_sanitisation_GFL_v1/rules.drl | 877 ++++-------------- 27 files changed, 850 insertions(+), 1332 deletions(-) rename redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/{services => graph}/DocumentPerformanceIntegrationTest.java (97%) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java index 791c8f81..76d925c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java @@ -9,11 +9,11 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; @@ -26,18 +26,23 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; -import lombok.RequiredArgsConstructor; - @Service -@RequiredArgsConstructor public class RedactionLogEntryAdapter { private static final double MATCH_THRESHOLD = 1; private final EntityCreationService entityCreationService; + @Autowired + public RedactionLogEntryAdapter(EntityEnrichmentService entityEnrichmentService) { + + entityCreationService = new EntityCreationService(entityEnrichmentService); + } + + public Stream toRedactionEntity(RedactionLog redactionLog, SemanticNode node) { List pageNumbers = redactionLog.getRedactionLogEntry().stream().flatMap(entry -> entry.getPositions().stream().map(Rectangle::getPage)).distinct().toList(); @@ -72,9 +77,7 @@ public class RedactionLogEntryAdapter { return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary()) .stream() - .map(boundary -> entityCreationService.byBoundary(boundary, "temp", EntityType.ENTITY, node)) - .filter(Optional::isPresent) - .map(Optional::get) + .map(boundary -> entityCreationService.forceByBoundary(boundary, "temp", EntityType.ENTITY, node)) .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT))); } @@ -100,10 +103,10 @@ public class RedactionLogEntryAdapter { private RedactionEntity createCorrectEntity(RedactionLogEntry redactionLogEntry, SemanticNode node, RedactionEntity closestEntity) { - RedactionEntity correctEntity = entityCreationService.byBoundary(closestEntity.getBoundary(), + RedactionEntity correctEntity = entityCreationService.forceByBoundary(closestEntity.getBoundary(), redactionLogEntry.getType(), redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY, - node).orElseThrow(); + node); String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0"; if (redactionLogEntry.isRedacted()) { correctEntity.apply(ruleIdentifier, redactionLogEntry.getReason(), redactionLogEntry.getLegalBasis()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java index b043e490..ee1767e2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java @@ -47,6 +47,12 @@ public final class MatchedRule implements Comparable { if (Objects.equals(ruleIdentifier.type(), "MAN")) { return -1; } + if (Objects.equals(otherRuleIdentifier.type(), "X")) { + return 1; + } + if (Objects.equals(ruleIdentifier.type(), "X")) { + return -1; + } } if (!Objects.equals(otherRuleIdentifier.unit(), getRuleIdentifier().unit())) { return otherRuleIdentifier.unit() - ruleIdentifier.unit(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java index 45ce83db..44af7bf9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java @@ -12,6 +12,18 @@ public interface MatchedRuleHolder { PriorityQueue getMatchedRuleList(); + boolean isIgnored(); + + + boolean isRemoved(); + + + void setIgnored(boolean ignored); + + + void setRemoved(boolean ignored); + + default boolean isApplied() { return getMatchedRule().isApplied(); @@ -24,18 +36,24 @@ public interface MatchedRuleHolder { } + default boolean isActive() { + + return !(isRemoved() || isIgnored()); + } + + default void apply(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) { if (legalBasis.isBlank() || legalBasis.isEmpty()) { throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); } - getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build()); + addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build()); } default void force(@NonNull String ruleIdentifier, String reason, String legalBasis) { - getMatchedRuleList().add(MatchedRule.builder() + addMatchedRule(MatchedRule.builder() .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) .reason(reason) .legalBasis(getLegalBasisOrPreviousLegalBasisOrPlaceHolder(legalBasis)) @@ -44,6 +62,26 @@ public interface MatchedRuleHolder { } + default void skip(@NonNull String ruleIdentifier, String reason) { + + addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build()); + } + + + default void remove(String ruleIdentifier, String reason) { + + addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build()); + setRemoved(true); + } + + + default void ignore(String ruleIdentifier, String reason) { + + addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build()); + setIgnored(true); + } + + private String getLegalBasisOrPreviousLegalBasisOrPlaceHolder(String legalBasis) { if (legalBasis == null || legalBasis.isBlank() || legalBasis.isEmpty()) { @@ -86,12 +124,6 @@ public interface MatchedRuleHolder { } - default void skip(@NonNull String ruleIdentifier, String reason) { - - getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build()); - } - - default void skipWithReferences(@NonNull String ruleIdentifier, String reason, Collection references) { getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).references(new HashSet<>(references)).build()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java index b6d8a282..5ddc15d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java @@ -94,12 +94,6 @@ public class RedactionEntity implements MatchedRuleHolder { } - public boolean isActive() { - - return !ignored && !removed; - } - - public void addIntersectingNode(SemanticNode containingNode) { intersectingNodes.add(containingNode); @@ -123,18 +117,6 @@ public class RedactionEntity implements MatchedRuleHolder { } - public void remove() { - - removed = true; - } - - - public void ignore() { - - ignored = true; - } - - public List getRedactionPositionsPerPage() { if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java index d2a9cc79..ed5b64c7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java @@ -55,24 +55,6 @@ public class Image implements GenericSemanticNode, MatchedRuleHolder { Set entities = new HashSet<>(); - public boolean isActive() { - - return !removed && !ignored; - } - - - public void ignore() { - - ignored = true; - } - - - public void remove() { - - removed = true; - } - - @Override public NodeType getType() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java index 22eff118..fe9d14a1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.n import static java.lang.String.format; import java.awt.geom.Rectangle2D; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -206,6 +207,36 @@ public interface SemanticNode { } + /** + * Checks whether this SemanticNode has any Entity of the provided types. + * Ignores Entity with ignored == true or removed == true. + * + * @param types an array of strings representing the types of entities to check for + * @return true, if this SemanticNode has at least one Entity of any of the provided types + */ + default boolean hasEntitiesOfAnyType(String... types) { + + return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type))); + } + + + /** + * Checks whether this SemanticNode has at least one Entity of each of the provided types. + * Ignores Entity with ignored == true or removed == true. + * + * @param types an array of strings representing the types of entities to check for + * @return true, if this SemanticNode has at least one Entity of each of the provided types + */ + default boolean hasEntitiesOfAllTypes(String... types) { + + return getEntities().stream() + .filter(RedactionEntity::isActive) + .map(RedactionEntity::getType) + .collect(Collectors.toUnmodifiableSet()) + .containsAll(Arrays.stream(types).toList()); + } + + /** * Returns a List of Entities in this SemanticNode which are of the provided type such as "CBI_author". * Ignores Entity with ignored == true or removed == true. @@ -232,6 +263,19 @@ public interface SemanticNode { } + /** + * Returns a List of Entities in this SemanticNode which have any of the provided types. + * Ignores Entity with the ignored flag set to true or the removed flag set to true. + * + * @param types A list of strings representing the types of entities to return + * @return List of RedactionEntities that match any of the provided types + */ + default List getEntitiesOfType(String... types) { + + return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList(); + } + + /** * Each AtomicTextBlock has an index on its page, this returns the number of the first AtomicTextBlock underneath this node. * If this node does not have any AtomicTexBlocks underneath it, e.g. an empty TableCell. It returns -1. @@ -278,21 +322,21 @@ public interface SemanticNode { * @param strings A List of Strings which the TextBlock might contain * @return true, if this node's TextBlock contains all strings */ - default boolean containsStrings(List strings) { + default boolean containsAllStrings(String... strings) { - return strings.stream().allMatch(this::containsString); + return Arrays.stream(strings).allMatch(this::containsString); } /** - * Checks whether this SemanticNode contains all the provided Strings ignoring case. + * Checks whether this SemanticNode contains any of the provided Strings. * - * @param string A String which the TextBlock might contain - * @return true, if this node's TextBlock contains the string ignoring case + * @param strings A List of Strings to check if they are contained in the TextBlock + * @return true, if this node's TextBlock contains any of the provided strings */ - default boolean containsStringIgnoreCase(String string) { + default boolean containsAnyString(String... strings) { - return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase()); + return Arrays.stream(strings).anyMatch(this::containsString); } @@ -308,15 +352,39 @@ public interface SemanticNode { } + /** + * Checks whether this SemanticNode contains all the provided Strings ignoring case. + * + * @param string A String which the TextBlock might contain + * @return true, if this node's TextBlock contains the string ignoring case + */ + default boolean containsStringIgnoreCase(String string) { + + return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase()); + } + + /** * Checks whether this SemanticNode contains any of the provided Strings ignoring case. * * @param strings A List of Strings which the TextBlock might contain * @return true, if this node's TextBlock contains any of the strings */ - default boolean containsAnyStringIgnoreCase(List strings) { + default boolean containsAnyStringIgnoreCase(String... strings) { - return strings.stream().anyMatch(this::containsStringIgnoreCase); + return Arrays.stream(strings).anyMatch(this::containsStringIgnoreCase); + } + + + /** + * Checks whether this SemanticNode contains any of the provided Strings ignoring case. + * + * @param strings A List of Strings which the TextBlock might contain + * @return true, if this node's TextBlock contains any of the strings + */ + default boolean containsAllStringsIgnoreCase(String... strings) { + + return Arrays.stream(strings).allMatch(this::containsStringIgnoreCase); } @@ -420,6 +488,17 @@ public interface SemanticNode { } + /** + * Returns the length of the text content in this Node's TextBlock. + * + * @return The length of the text content + */ + default int length() { + + return getBoundary().length(); + } + + /** * If this Node is a Leaf it will calculate the boundingBox of its LeafTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children. * If called on the Document, it will return the cropbox of each page diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index f7b40426..ef0df4c2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -15,7 +15,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.springframework.stereotype.Service; +import org.kie.api.runtime.KieSession; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; @@ -37,11 +37,18 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@Service @RequiredArgsConstructor public class EntityCreationService { private final EntityEnrichmentService entityEnrichmentService; + private final KieSession kieSession; + + + public EntityCreationService(EntityEnrichmentService entityEnrichmentService) { + + this.entityEnrichmentService = entityEnrichmentService; + this.kieSession = null; + } public Stream betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { @@ -138,22 +145,6 @@ public class EntityCreationService { } - public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation, - String type, - EntityType entityType, - SemanticNode node, - boolean isDossierDictionaryEntry) { - - searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary()) - .stream() - .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) - .map(bounds -> forceByBoundary(bounds, type, entityType, node)) - .peek(entity -> entity.setDictionaryEntry(true)) - .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry)) - .forEach(entity -> entity.addEngine(Engine.DICTIONARY)); - } - - public Stream lineAfterStrings(List strings, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.getTextBlock(); @@ -181,6 +172,21 @@ public class EntityCreationService { } + public Optional semanticNodeAfterString(SemanticNode semanticNode, String string, String type, EntityType entityType) { + + var textBlock = semanticNode.getTextBlock(); + int startIndex = Math.min(textBlock.indexOf(string), 0); + var boundary = new Boundary(startIndex, semanticNode.getBoundary().end()); + if (boundary.length() > 0) { + boundary = new Boundary(boundary.start(), boundary.end() - 1); + } + if (!isValidEntityBoundary(textBlock, boundary)) { + return Optional.empty(); + } + return byBoundary(boundary, type, entityType, semanticNode); + } + + public Stream byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, SemanticNode node) { return byRegexWithLineBreaks(regexPattern, type, entityType, 0, node); @@ -301,13 +307,14 @@ public class EntityCreationService { /** * Creates a redaction entity based on the given boundary, type, entity type, and semantic node. - * If the document already contains an equal redaction entity, then en empty Optional is returned. + * If the document already contains an equal redaction entity, then the original Entity is returned. + * Also inserts the Entity into the kieSession. * * @param boundary The boundary of the redaction entity. * @param type The type of the redaction entity. * @param entityType The entity type of the redaction entity. * @param node The semantic node to associate with the redaction entity. - * @return An Optional containing the redaction entity, or an empty Optional if the entity already exists. + * @return An Optional containing the redaction entity, or the previous entity if the entity already exists. */ public Optional byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { @@ -317,10 +324,13 @@ public class EntityCreationService { Boundary trimmedBoundary = boundary.trim(node.getTextBlock()); RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType); if (node.getEntities().contains(entity)) { - return Optional.empty(); + return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngine(Engine.RULE)).findAny(); } addEntityToGraph(entity, node); entity.addEngine(Engine.RULE); + if (kieSession != null) { + kieSession.insert(entity); + } return Optional.of(entity); } @@ -356,6 +366,7 @@ public class EntityCreationService { mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved)); addEntityToGraph(mergedEntity, node); + kieSession.insert(mergedEntity); return mergedEntity; } @@ -391,7 +402,7 @@ public class EntityCreationService { } - private boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) { + public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) { return boundary.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, boundary); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java index 3b919211..d9bf3bcf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java @@ -3,16 +3,15 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.service import java.awt.geom.Rectangle2D; import java.util.NoSuchElementException; -import org.springframework.stereotype.Service; - import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations; import lombok.RequiredArgsConstructor; -@Service @RequiredArgsConstructor public class ManualRedactionApplicationService { @@ -47,6 +46,16 @@ public class ManualRedactionApplicationService { } + public void resizeImage(Image image, ManualResizeRedaction manualResizeRedaction) { + + if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) { + return; + } + var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions().stream().map(ManualRedactionApplicationService::toRectangle2D).toList()); + image.setPosition(bBox); + } + + private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) { return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 0f6a5beb..2a658a8c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -138,7 +138,7 @@ public class AnalyzeService { dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - log.info("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); @@ -180,8 +180,10 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + // not yet ready for reanalysis if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) { return analyze(analyzeRequest); @@ -207,10 +209,14 @@ public class AnalyzeService { NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - log.info("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); + log.info("Updated Rules to version {} for file {} in dossier {}", + droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()), + analyzeRequest.getFileId(), + analyzeRequest.getDossierId()); + + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index b6fafe06..fb8ea98d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -26,6 +26,7 @@ import com.iqser.red.service.redaction.v1.server.exception.RulesValidationExcept import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; @@ -46,8 +47,7 @@ public class DroolsExecutionService { RulesClient rulesClient; Map kieContainers = new HashMap<>(); Map rulesVersionPerDossierTemplateId = new HashMap<>(); - EntityCreationService entityCreationService; - ManualRedactionApplicationService manualRedactionApplicationService; + EntityEnrichmentService entityEnrichmentService; NerEntitiesAdapter nerEntitiesAdapter; @@ -84,6 +84,9 @@ public class DroolsExecutionService { NerEntities nerEntities) { KieSession kieSession = kieContainer.newKieSession(); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); + ManualRedactionApplicationService manualRedactionApplicationService = new ManualRedactionApplicationService(entityCreationService); + kieSession.setGlobal("document", document); kieSession.setGlobal("entityCreationService", entityCreationService); kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 2e352ccf..f1357346 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -9,12 +9,15 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -28,7 +31,7 @@ import lombok.extern.slf4j.Slf4j; public class EntityRedactionService { DroolsExecutionService droolsExecutionService; - EntityCreationService entityCreationService; + EntityEnrichmentService entityEnrichmentService; public Set addRuleEntities(Dictionary dictionary, Document document, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) { @@ -71,14 +74,27 @@ public class EntityRedactionService { public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { for (var model : dictionary.getDictionaryModels()) { - entityCreationService.bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary()); - entityCreationService.bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); - entityCreationService.bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), - model.getType(), - EntityType.FALSE_RECOMMENDATION, - node, - model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); } } + + public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation, + String type, + EntityType entityType, + SemanticNode node, + boolean isDossierDictionaryEntry) { + + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary()) + .stream() + .filter(boundary -> entityCreationService.isValidEntityBoundary(node.getTextBlock(), boundary)) + .map(bounds -> entityCreationService.forceByBoundary(bounds, type, entityType, node)) + .peek(entity -> entity.setDictionaryEntry(true)) + .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry)) + .forEach(entity -> entity.addEngine(Engine.DICTIONARY)); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java index 3cf977b6..405af603 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java @@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -35,7 +36,7 @@ import lombok.extern.slf4j.Slf4j; public class ManualRedactionSurroundingTextService { private final RedactionStorageService redactionStorageService; - private final EntityCreationService entityCreationService; + private final EntityEnrichmentService entityEnrichmentService; @Timed("redactmanager_surroundingTextAnalysis") @@ -73,6 +74,7 @@ public class ManualRedactionSurroundingTextService { private Pair findSurroundingText(SemanticNode node, String value, List toFindPositions) { + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); Set entities = RedactionSearchUtility.findBoundariesByString(value, node.getTextBlock()) .stream() .map(boundary -> entityCreationService.forceByBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node)) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index da78f73f..1712f5ec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -83,7 +83,7 @@ public class RedactionChangeLogService { currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries); - log.info("Change computation took: {}", System.currentTimeMillis() - start); + log.debug("Change computation took: {}", System.currentTimeMillis() - start); return new RedactionLogChanges(currentRedactionLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index b43e83f5..c6549313 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -42,6 +42,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { @Test + @Disabled public void titleExtraction() throws IOException { AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/A13617AV/425_F.1.1.1 - A13617AV - Acute Oral Toxicity Study.pdf"); @@ -65,6 +66,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { @Test + @Disabled public void tableWithEmptyCols() throws IOException { // FIXME TableNodeFactory: 36, why has table no rows/cols here. @@ -89,8 +91,8 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { } - @Test + @Disabled public void testTopOfPage13InNotHeader() throws IOException { // Fix In BodyTextFrameService destroys header detection in files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf @@ -116,7 +118,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { } - @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 260960b5..1bd38749 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.time.OffsetDateTime; import java.util.List; import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -30,6 +31,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; @@ -103,29 +106,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { System.out.println("Finished analysis"); var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var publishedInformationEntry1 = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> entry.getType().equals("published_information")) - .filter(entry -> entry.getValue().equals("Oxford University Press")) - .findFirst() - .orElseThrow(); - - var asyaLyon1 = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> entry.getType().equals("CBI_author")) - .filter(entry -> entry.getValue().equals("Asya Lyon")) - .filter(entry -> entry.getSectionNumber() == publishedInformationEntry1.getSectionNumber()) - .findFirst() - .orElseThrow(); + var publishedInformationEntry1 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); + var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow(); // assertFalse(asyaLyon1.isRedacted()); - var idRemoval = IdRemoval.builder() - .requestDate(OffsetDateTime.now()) - .annotationId(publishedInformationEntry1.getId()) - .status(AnnotationStatus.APPROVED) - .fileId(TEST_FILE_ID) - .build(); + var idRemoval = buildIdRemoval(publishedInformationEntry1.getId()); var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build(); request.setManualRedactions(manualRedactions); @@ -133,20 +119,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var publishedInformationEntry2 = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> entry.getType().equals("published_information")) - .filter(entry -> entry.getValue().equals("Oxford University Press")) - .findFirst() - .orElseThrow(); - - var asyaLyon2 = redactionLog.getRedactionLogEntry() - .stream() - .filter(entry -> entry.getType().equals("CBI_author")) - .filter(entry -> entry.getValue().equals("Asya Lyon")) - .filter(entry -> entry.getSectionNumber() == publishedInformationEntry2.getSectionNumber()) - .findFirst() - .orElseThrow(); + var publishedInformationEntry2 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow(); + var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow(); assertTrue(asyaLyon2.isRedacted()); @@ -160,4 +134,50 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { } + + private Stream findEntityByTypeAndValueAndSectionNumber(RedactionLog redactionLog, String type, String value, int sectionNumber) { + + return redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getType().equals(type)) + .filter(entry -> entry.getValue().equals(value)) + .filter(entry -> entry.getSectionNumber() == sectionNumber); + } + + + private static Stream findEntityByTypeAndValue(RedactionLog redactionLog, String type, String value) { + + return redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value)); + } + + + @Test + public void noEndlessLoopsTest() { + + AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf"); + System.out.println("Start Full integration test"); + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + System.out.println("Finished structure analysis"); + AnalyzeResult result = analyzeService.analyze(request); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> e.getMatchedRule().startsWith("CBI.16")) + .findAny() + .orElseThrow(); + IdRemoval removal = buildIdRemoval(desireeEtAl.getId()); + request.setManualRedactions(ManualRedactions.builder().idsToRemove(Set.of(removal)).build()); + + analyzeService.reanalyze(request); + + var redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + assertTrue(findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").noneMatch(e -> e.getMatchedRule().startsWith("CBI.16"))); + + } + + + private static IdRemoval buildIdRemoval(String id) { + + return IdRemoval.builder().annotationId(id).requestDate(OffsetDateTime.now()).status(AnnotationStatus.APPROVED).fileId(TEST_FILE_ID).build(); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 91b9577a..0936a995 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -75,6 +75,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.StorageAutoConfiguration; @@ -89,9 +90,8 @@ import lombok.SneakyThrows; public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/rules.drl"); - @Autowired - private EntityCreationService entityCreationService; + private EntityEnrichmentService entityEnrichmentService; @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @@ -702,6 +702,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."; + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); RedactionEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get(); String idToResize = redactionLog.getRedactionLogEntry() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java index 82be8fa9..7b9958ea 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java @@ -2,11 +2,18 @@ package com.iqser.red.service.redaction.v1.server.document.graph; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.wildfly.common.Assert.assertTrue; import java.util.List; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.kie.api.runtime.KieSession; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; @@ -23,12 +30,25 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest { @Autowired + private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; + @Mock + private KieSession kieSession; + + + @BeforeEach + public void createEntityCreationService() { + + MockitoAnnotations.initMocks(this); + entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); + } + @Test public void assertCollectAllEntitiesWorks() { @@ -50,8 +70,9 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra Document document = buildGraph("files/new/crafted document.pdf"); String type = "CBI_author"; assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isPresent()); - assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isEmpty()); + assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isPresent()); assertEquals(1, document.getEntities().size()); + verify(kieSession, times(1)).insert(any(RedactionEntity.class)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index cd912a75..fd964cb2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.services; +package com.iqser.red.service.redaction.v1.server.document.graph; import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import static org.mockito.Mockito.when; @@ -31,7 +31,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; -import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; @@ -40,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.PdfVisualisationUtility; import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; @@ -60,6 +60,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration private DictionaryService dictionaryService; @Autowired + private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; @Autowired @@ -93,6 +94,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration @BeforeEach public void stubClients() { + entityCreationService = new EntityCreationService(entityEnrichmentService); TenantContext.setTenantId("redaction"); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java index 1d0b04d9..ec0d3674 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.kie.api.KieServices; import org.kie.api.builder.KieBuilder; @@ -36,6 +37,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; @Import(ManualResizeRedactionIntegrationTest.TestConfiguration.class) @@ -44,9 +46,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati private static final String RULES = "drools/manual_redaction_rules.drl"; @Autowired + private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; - - @Autowired private ManualRedactionApplicationService manualRedactionApplicationService; @Qualifier("kieContainer") @@ -74,6 +75,14 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati } + @BeforeEach + public void createServices() { + + entityCreationService = new EntityCreationService(entityEnrichmentService); + manualRedactionApplicationService = new ManualRedactionApplicationService(entityCreationService); + } + + @Test public void manualResizeRedactionTest() { @@ -95,6 +104,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati KieSession kieSession = kieContainer.newKieSession(); kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService); + kieSession.insert(document); + document.streamAllSubNodes().forEach(kieSession::insert); kieSession.insert(entity); kieSession.insert(manualResizeRedaction); kieSession.fireAllRules(); @@ -131,6 +142,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService); kieSession.insert(entity); kieSession.insert(manualForceRedaction); + kieSession.insert(document); + document.streamAllSubNodes().forEach(kieSession::insert); kieSession.fireAllRules(); kieSession.dispose(); @@ -159,6 +172,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati KieSession kieSession = kieContainer.newKieSession(); kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService); + kieSession.insert(document); + document.streamAllSubNodes().forEach(kieSession::insert); kieSession.insert(entity); kieSession.insert(idRemoval); kieSession.fireAllRules(); @@ -189,6 +204,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati KieSession kieSession = kieContainer.newKieSession(); kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService); + kieSession.insert(document); + document.streamAllSubNodes().forEach(kieSession::insert); kieSession.insert(entity); kieSession.insert(idRemoval); kieSession.insert(manualForceRedaction); @@ -220,6 +237,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService); kieSession.insert(entity); kieSession.insert(idRemoval); + kieSession.insert(document); + document.streamAllSubNodes().forEach(kieSession::insert); kieSession.fireAllRules(); kieSession.dispose(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index d428ccc4..437454ef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -13,6 +13,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.pdfbox.pdmodel.PDDocument; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; @@ -26,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.PdfVisualisationUtility; import lombok.SneakyThrows; @@ -37,6 +39,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { @Autowired private NerEntitiesAdapter nerEntitiesAdapter; @Autowired + private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; @@ -48,6 +51,13 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { } + @BeforeEach + public void createEntityCreationService() { + + entityCreationService = new EntityCreationService(entityEnrichmentService); + } + + @Test @SneakyThrows public void testGetNerEntities() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 09c25230..0c1e10e7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -64,10 +64,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" then /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */ entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.skip("SYN.1.0", ""); - insert(entity); - }); + .forEach(entity -> entity.skip("SYN.1.0", "")); end @@ -115,10 +112,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author" $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied()) then entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document) - .ifPresent(falsePositive -> { - falsePositive.skip("CBI.2.0", "Genitive Author found"); - insert(falsePositive); - }); + .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); end @@ -166,10 +160,7 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)" @@ -182,10 +173,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -200,10 +188,7 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)" @@ -216,10 +201,7 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -245,7 +227,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" .forEach(entity -> { entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); - insert(entity); }); end @@ -258,7 +239,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(entity); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); }); end @@ -270,10 +250,7 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) then entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"); - insert(entity); - }); + .forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section")); end rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon" @@ -281,10 +258,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with $section: Section(!hasTables(), containsString("Species:"), containsString("Source:")) then entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"); - insert(entity); - }); + .forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section")); end @@ -299,7 +273,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC .forEach(laboratoryEntity -> { laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); }); end @@ -313,7 +286,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC .forEach(laboratoryEntity -> { laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); }); end @@ -345,10 +317,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> { - emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(emailEntity); - }); + .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" @@ -357,10 +326,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> { - emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(emailEntity); - }); + .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -380,10 +346,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" @@ -401,10 +364,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -415,10 +375,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)" @@ -427,10 +384,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" @@ -439,10 +393,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)" @@ -451,10 +402,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -466,7 +414,7 @@ rule "ETC.0.0: Purity Hint" $section: Section(containsStringIgnoreCase("purity")) then entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) - .forEach(hint -> hint.skip("ETC.0.0", "")); + .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end @@ -479,14 +427,6 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "ETC.2.0: Redact signatures (vertebrate study)" - when - FileAttribute(label == "Vertebrate Study", value == "Yes") - $signature: Image(imageType == ImageType.SIGNATURE) - then - $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - // Rule unit: ETC.3 rule "ETC.3.0: Redact logos (vertebrate study)" @@ -512,7 +452,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi not FileAttribute(label == "Confidentiality", value == "confidential") $dossierRedaction: RedactionEntity(type == "dossier_redaction") then - $dossierRedaction.setIgnored(true); + $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); update($dossierRedaction); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end @@ -527,8 +467,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) - .forEach(entity -> insert(entity)); + .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -540,10 +479,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) .map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) - .forEach(entity -> { - entity.addEngine(Engine.NER); - insert(entity); - }); + .forEach(entity -> entity.addEngine(Engine.NER)); end @@ -571,7 +507,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.setIgnored(true); + $entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction"); update($entityToBeRemoved); retract($idRemoval); $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); @@ -584,7 +520,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $imageEntityToBeRemoved: Image($id == id) then - $imageEntityToBeRemoved.setIgnored(true); + $imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction"); update($imageEntityToBeRemoved); retract($idRemoval); update($imageEntityToBeRemoved.getParent()); @@ -593,6 +529,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" + no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) @@ -604,7 +541,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); - retract($force); end @@ -628,10 +564,10 @@ rule "MAN.3.0: Apply image recategorization" rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when - $larger: RedactionEntity($type: type, $entityType: entityType) - $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + $larger: RedactionEntity($type: type, $entityType: entityType, isActive()) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $contained.remove(); + $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end @@ -640,15 +576,14 @@ rule "X.0.0: remove Entity contained by Entity of same type" rule "X.1.0: merge intersecting Entities of same type" salience 64 when - $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) - $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) + $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) + $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $first.remove(); - $second.remove(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + $first.remove("X.1.0", "merge intersecting Entities of same type"); + $second.remove("X.1.0", "merge intersecting Entities of same type"); retract($first); retract($second); - insert(mergedEntity); mergedEntity.getIntersectingNodes().forEach(node -> update(node)); end @@ -657,11 +592,11 @@ rule "X.1.0: merge intersecting Entities of same type" rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) - $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive()) + $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove(); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); retract($entity) end @@ -670,10 +605,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) - $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive()) + $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.remove(); + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); end @@ -682,11 +617,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.addEngines($recommendation.getEngines()); - $recommendation.remove(); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); retract($recommendation); end @@ -695,10 +630,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: RedactionEntity(entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity(entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.remove(); + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); end @@ -707,11 +642,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" salience 32 when - $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) + $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove(); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); retract($lowerRank); end @@ -738,6 +673,5 @@ rule "LDS.0.0: run local dictionary search" when DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then - entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) - .forEach(entity -> insert(entity)); + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document).toList(); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index d7115aae..6be15ae8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -58,7 +58,7 @@ query "getFileAttributes" //------------------------------------ Syngenta specific rules ------------------------------------ // Rule unit: SYN.0 -rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" +rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(containsString("CTL/") || containsString("BL/")) @@ -66,10 +66,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" Stream.concat( entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section), entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) - ).forEach(entity -> { - entity.skip("SYN.0.0", "hint_only"); - insert(entity); - }); + ).forEach(entity -> entity.skip("SYN.0.0", "hint_only")); end @@ -80,10 +77,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" then /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */ entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.skip("SYN.1.0", ""); - insert(entity); - }); + .forEach(entity -> entity.skip("SYN.1.0", "")); end @@ -131,10 +125,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author" $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied()) then entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document) - .ifPresent(falsePositive -> { - falsePositive.skip("CBI.2.0", "Genitive Author found"); - insert(falsePositive); - }); + .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); end @@ -385,10 +376,7 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)" @@ -401,10 +389,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -419,10 +404,7 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)" @@ -435,10 +417,7 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -466,10 +445,7 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author" .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.skip("CBI.12.0", "Author(s) header found"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found")); end rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No" @@ -478,9 +454,7 @@ rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \ then $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No")) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> { - authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study"); - }); + .forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study")); end rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes" @@ -489,9 +463,7 @@ rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vert then $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> { - authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); + .forEach(authorEntity -> authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -501,7 +473,7 @@ rule "CBI.13.0: Ignore CBI Address Recommendations" not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION) then - $entity.remove(); + $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); retract($entity) end @@ -533,12 +505,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv containsStringIgnoreCase($keyword)) then entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section) - .forEach(keywordEntity -> insert(keywordEntity)); + .toList(); $section.getEntitiesOfType(List.of($keyword, $residueKeyword)) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword" @@ -556,13 +526,11 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio $table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) then entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table) - .forEach(keywordEntity -> insert(keywordEntity)); + .toList(); $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword)) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -577,7 +545,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" .forEach(entity -> { entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); - insert(entity); }); end @@ -590,7 +557,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(entity); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); }); end @@ -602,10 +568,7 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) then entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"); - insert(entity); - }); + .forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section")); end rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon" @@ -613,10 +576,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with $section: Section(!hasTables(), containsString("Species:"), containsString("Source:")) then entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"); - insert(entity); - }); + .forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section")); end @@ -632,11 +592,10 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" then entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") .ifPresent(expandedEntity -> { - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.remove(); + expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList()); + $entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials"); retract($entityToExpand); - insert(expandedEntity); - }); + }); end @@ -647,11 +606,10 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> { - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.remove(); + expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList()); + $entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix"); retract($entityToExpand); - insert(expandedEntity); - }); + }); end @@ -666,7 +624,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC .forEach(laboratoryEntity -> { laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); }); end @@ -680,7 +637,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC .forEach(laboratoryEntity -> { laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); }); end @@ -712,10 +668,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> { - emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(emailEntity); - }); + .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" @@ -724,10 +677,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> { - emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(emailEntity); - }); + .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -747,10 +697,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" @@ -768,10 +715,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -782,10 +726,7 @@ rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> { - entity.apply("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" @@ -794,10 +735,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> { - entity.apply("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -827,10 +765,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)" @@ -858,10 +793,7 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end @@ -876,10 +808,7 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)" @@ -892,10 +821,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -909,10 +835,7 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> { - contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.6.1: redact line between contact keywords" @@ -924,10 +847,7 @@ rule "PII.6.1: redact line between contact keywords" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> { - contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -947,10 +867,7 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> { - entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)" @@ -968,10 +885,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> { - entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -991,10 +905,7 @@ rule "PII.8.0: Redact contact information if producer is found" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> { - entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end rule "PII.8.1: Redact contact information if producer is found" @@ -1012,10 +923,7 @@ rule "PII.8.1: Redact contact information if producer is found" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> { - entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -1026,10 +934,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)" @@ -1038,10 +943,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" @@ -1050,10 +952,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)" @@ -1062,10 +961,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -1075,10 +971,7 @@ rule "PII.10.0: Redact study director abbreviation" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> { - entity.apply("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -1088,10 +981,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -1101,10 +991,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" $entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> { - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - insert(expandedEntity); - }); + .ifPresent(expandedEntity -> expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList())); end @@ -1116,7 +1003,7 @@ rule "ETC.0.0: Purity Hint" $section: Section(containsStringIgnoreCase("purity")) then entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) - .forEach(hint -> hint.skip("ETC.0.0", "")); + .forEach(hint -> hint.skip("ETC.0.0", "hint only")); end @@ -1139,7 +1026,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "ETC.2.0: Redact signatures (vertebrate study)" +rule "ETC.2.1: Redact signatures (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value == "Yes") $signature: Image(imageType == ImageType.SIGNATURE) @@ -1181,7 +1068,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi not FileAttribute(label == "Confidentiality", value == "confidential") $dossierRedaction: RedactionEntity(type == "dossier_redaction") then - $dossierRedaction.setIgnored(true); + $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); update($dossierRedaction); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end @@ -1196,10 +1083,7 @@ rule "ETC.6.0: Redact CAS Number" .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -1242,8 +1126,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) - .forEach(entity -> insert(entity)); + .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -1255,10 +1138,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) .map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) - .forEach(entity -> { - entity.addEngine(Engine.NER); - insert(entity); - }); + .forEach(entity -> entity.addEngine(Engine.NER)); end @@ -1270,9 +1150,7 @@ rule "AI.2.0: add all NER Entities of any type except CBI_author" then nerEntities.getNerEntityList().stream() .filter(nerEntity -> !nerEntity.type().equals("CBI_author")) - .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document)) - .forEach(entity -> insert(entity)); - + .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document)); end @@ -1291,6 +1169,18 @@ rule "MAN.0.0: Apply manual resize redaction" $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId) + $imageToBeResized: Image(id == $id) + then + manualRedactionApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); + end + // Rule unit: MAN.1 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" @@ -1300,7 +1190,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.setIgnored(true); + $entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction"); update($entityToBeRemoved); retract($idRemoval); $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); @@ -1313,7 +1203,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $imageEntityToBeRemoved: Image($id == id) then - $imageEntityToBeRemoved.setIgnored(true); + $imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction"); update($imageEntityToBeRemoved); retract($idRemoval); update($imageEntityToBeRemoved.getParent()); @@ -1322,6 +1212,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" + no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) @@ -1333,7 +1224,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); - retract($force); end rule "MAN.2.1: Apply force redaction to images" @@ -1370,10 +1260,10 @@ rule "MAN.3.0: Apply image recategorization" rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when - $larger: RedactionEntity($type: type, $entityType: entityType) - $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + $larger: RedactionEntity($type: type, $entityType: entityType, isActive()) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $contained.remove(); + $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end @@ -1382,15 +1272,14 @@ rule "X.0.0: remove Entity contained by Entity of same type" rule "X.1.0: merge intersecting Entities of same type" salience 64 when - $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) - $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) + $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) + $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $first.remove(); - $second.remove(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + $first.remove("X.1.0", "merge intersecting Entities of same type"); + $second.remove("X.1.0", "merge intersecting Entities of same type"); retract($first); retract($second); - insert(mergedEntity); mergedEntity.getIntersectingNodes().forEach(node -> update(node)); end @@ -1399,11 +1288,11 @@ rule "X.1.0: merge intersecting Entities of same type" rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) - $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive()) + $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove(); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); retract($entity) end @@ -1412,10 +1301,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) - $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive()) + $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.remove(); + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); end @@ -1424,11 +1313,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.addEngines($recommendation.getEngines()); - $recommendation.remove(); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); retract($recommendation); end @@ -1437,10 +1326,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: RedactionEntity(entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity(entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.remove(); + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); end @@ -1449,11 +1338,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" salience 32 when - $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) + $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove(); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); retract($lowerRank); end @@ -1480,6 +1369,5 @@ rule "LDS.0.0: run local dictionary search" when DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then - entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) - .forEach(entity -> insert(entity)); + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document).toList(); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index afbfd462..646acf52 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -58,6 +58,19 @@ rule "MAN.0.0: Apply manual resize redaction" manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); + $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId) + $imageToBeResized: Image(id == $id) + then + manualRedactionApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); end @@ -65,44 +78,71 @@ rule "MAN.0.0: Apply manual resize redaction" rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.setIgnored(true); + $entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction"); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $imageEntityToBeRemoved: Image($id == id) then - $imageEntityToBeRemoved.setIgnored(true); + $imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction"); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); end // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" + no-loop true salience 128 when - ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) $entityToForce: RedactionEntity(matchesAnnotationId($id)) then - $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.setRemoved(false); + $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + update($entityToForce); + $entityToForce.getIntersectingNodes().forEach(node -> update(node)); end +rule "MAN.2.1: Apply force redaction to images" + no-loop true + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $imageToForce: Image(id == $id) + then + $imageToForce.force("MAN.2.0", "Forced redaction", $legalBasis); + $imageToForce.setRemoved(false); + $imageToForce.setIgnored(false); + update($imageToForce); + update($imageToForce.getParent()); + end // Rule unit: MAN.3 rule "MAN.3.0: Apply image recategorization" salience 128 when - ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) - $image: Image($id == id) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) + $imageToBeRecategorized: Image($id == id) then - $image.setImageType(ImageType.fromString($imageType)); + $imageToBeRecategorized.setImageType(ImageType.fromString($imageType)); + update($imageToBeRecategorized); + update($imageToBeRecategorized.getParent()); + retract($recategorization); end //------------------------------------ Local dictionary search rules ------------------------------------ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 869cbee8..922db6a9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -58,7 +58,7 @@ query "getFileAttributes" //------------------------------------ Syngenta specific rules ------------------------------------ // Rule unit: SYN.0 -rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" +rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $section: Section(containsString("CTL/") || containsString("BL/")) @@ -66,10 +66,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" Stream.concat( entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section), entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) - ).forEach(entity -> { - entity.skip("SYN.0.0", "hint_only"); - insert(entity); - }); + ).forEach(entity -> entity.skip("SYN.0.0", "hint_only")); end @@ -249,10 +246,7 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)" @@ -265,10 +259,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -296,10 +287,7 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author" .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.skip("CBI.12.0", "Author(s) header found"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found")); end rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No" @@ -308,9 +296,7 @@ rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \ then $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No")) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> { - authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study"); - }); + .forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study")); end rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes" @@ -319,9 +305,7 @@ rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vert then $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> { - authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); + .forEach(authorEntity -> authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -352,12 +336,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv containsStringIgnoreCase($keyword)) then entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section) - .forEach(keywordEntity -> insert(keywordEntity)); + .toList(); $section.getEntitiesOfType(List.of($keyword, $residueKeyword)) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword" @@ -375,13 +357,11 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio $table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) then entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table) - .forEach(keywordEntity -> insert(keywordEntity)); + .toList(); $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword)) .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(redactionEntity -> { - redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -396,7 +376,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" .forEach(entity -> { entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); - insert(entity); }); end @@ -409,7 +388,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(entity); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); }); end @@ -421,10 +399,7 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) then entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"); - insert(entity); - }); + .forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section")); end rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon" @@ -432,10 +407,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with $section: Section(!hasTables(), containsString("Species:"), containsString("Source:")) then entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"); - insert(entity); - }); + .forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section")); end @@ -451,11 +423,10 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" then entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") .ifPresent(expandedEntity -> { - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.remove(); + expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList()); + $entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials"); retract($entityToExpand); - insert(expandedEntity); - }); + }); end @@ -466,11 +437,10 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> { - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.remove(); + expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList()); + $entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix"); retract($entityToExpand); - insert(expandedEntity); - }); + }); end @@ -485,7 +455,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC .forEach(laboratoryEntity -> { laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); }); end @@ -499,7 +468,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC .forEach(laboratoryEntity -> { laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); }); end @@ -531,10 +499,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> { - emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(emailEntity); - }); + .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" @@ -543,10 +508,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> { - emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(emailEntity); - }); + .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -576,10 +538,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)" @@ -607,10 +566,7 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end @@ -624,10 +580,7 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> { - contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.6.1: redact line between contact keywords" @@ -639,10 +592,7 @@ rule "PII.6.1: redact line between contact keywords" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> { - contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(contactEntity); - }); + .forEach(contactEntity -> contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -662,10 +612,7 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> { - entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)" @@ -683,10 +630,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> { - entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -706,10 +650,7 @@ rule "PII.8.0: Redact contact information if producer is found" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> { - entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end rule "PII.8.1: Redact contact information if producer is found" @@ -727,10 +668,7 @@ rule "PII.8.1: Redact contact information if producer is found" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> { - entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(entity); - }); + .forEach(entity -> entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -741,10 +679,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)" @@ -753,10 +688,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" @@ -765,10 +697,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)" @@ -777,10 +706,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -790,10 +716,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end @@ -803,10 +726,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" $entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> { - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - insert(expandedEntity); - }); + .ifPresent(expandedEntity -> expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList())); end @@ -831,7 +751,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "ETC.2.0: Redact signatures (vertebrate study)" +rule "ETC.2.1: Redact signatures (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value == "Yes") $signature: Image(imageType == ImageType.SIGNATURE) @@ -873,7 +793,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi not FileAttribute(label == "Confidentiality", value == "confidential") $dossierRedaction: RedactionEntity(type == "dossier_redaction") then - $dossierRedaction.setIgnored(true); + $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); update($dossierRedaction); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end @@ -888,10 +808,7 @@ rule "ETC.6.0: Redact CAS Number" .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)")); end @@ -934,8 +851,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) - .forEach(entity -> insert(entity)); + .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -947,10 +863,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) .map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) - .forEach(entity -> { - entity.addEngine(Engine.NER); - insert(entity); - }); + .forEach(entity -> entity.addEngine(Engine.NER)); end @@ -969,6 +882,18 @@ rule "MAN.0.0: Apply manual resize redaction" $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId) + $imageToBeResized: Image(id == $id) + then + manualRedactionApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); + end + // Rule unit: MAN.1 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" @@ -978,7 +903,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.setIgnored(true); + $entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction"); update($entityToBeRemoved); retract($idRemoval); $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); @@ -991,7 +916,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $imageEntityToBeRemoved: Image($id == id) then - $imageEntityToBeRemoved.setIgnored(true); + $imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction"); update($imageEntityToBeRemoved); retract($idRemoval); update($imageEntityToBeRemoved.getParent()); @@ -1000,6 +925,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" + no-loop true salience 128 when $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) @@ -1011,7 +937,20 @@ rule "MAN.2.0: Apply force redaction" $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); update($entityToForce); $entityToForce.getIntersectingNodes().forEach(node -> update(node)); - retract($force); + end + +rule "MAN.2.1: Apply force redaction to images" + no-loop true + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $imageToForce: Image(id == $id) + then + $imageToForce.force("MAN.2.0", "Forced redaction", $legalBasis); + $imageToForce.setRemoved(false); + $imageToForce.setIgnored(false); + update($imageToForce); + update($imageToForce.getParent()); end @@ -1035,10 +974,10 @@ rule "MAN.3.0: Apply image recategorization" rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when - $larger: RedactionEntity($type: type, $entityType: entityType) - $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + $larger: RedactionEntity($type: type, $entityType: entityType, isActive()) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $contained.remove(); + $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end @@ -1047,15 +986,14 @@ rule "X.0.0: remove Entity contained by Entity of same type" rule "X.1.0: merge intersecting Entities of same type" salience 64 when - $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) - $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) + $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) + $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $first.remove(); - $second.remove(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + $first.remove("X.1.0", "merge intersecting Entities of same type"); + $second.remove("X.1.0", "merge intersecting Entities of same type"); retract($first); retract($second); - insert(mergedEntity); mergedEntity.getIntersectingNodes().forEach(node -> update(node)); end @@ -1064,11 +1002,11 @@ rule "X.1.0: merge intersecting Entities of same type" rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) - $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive()) + $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove(); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); retract($entity) end @@ -1077,10 +1015,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) - $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive()) + $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.remove(); + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); end @@ -1089,11 +1027,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.addEngines($recommendation.getEngines()); - $recommendation.remove(); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); retract($recommendation); end @@ -1102,10 +1040,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: RedactionEntity(entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity(entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.remove(); + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); end @@ -1114,11 +1052,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" salience 32 when - $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) + $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove(); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); retract($lowerRank); end @@ -1145,6 +1083,5 @@ rule "LDS.0.0: run local dictionary search" when DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then - entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) - .forEach(entity -> insert(entity)); + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document).toList(); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 746e718f..00729943 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -80,16 +80,18 @@ rule "Always redact PII" $cbiAuthor.apply("PII.0.0", "PII found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end + + //------------------------------------ Entity merging rules ------------------------------------ // Rule unit: X.0 rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when - $larger: RedactionEntity($type: type, $entityType: entityType) - $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + $larger: RedactionEntity($type: type, $entityType: entityType, isActive()) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $contained.remove(); + $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end @@ -98,15 +100,14 @@ rule "X.0.0: remove Entity contained by Entity of same type" rule "X.1.0: merge intersecting Entities of same type" salience 64 when - $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) - $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) + $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) + $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $first.remove(); - $second.remove(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + $first.remove("X.1.0", "merge intersecting Entities of same type"); + $second.remove("X.1.0", "merge intersecting Entities of same type"); retract($first); retract($second); - insert(mergedEntity); mergedEntity.getIntersectingNodes().forEach(node -> update(node)); end @@ -115,11 +116,11 @@ rule "X.1.0: merge intersecting Entities of same type" rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) - $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive()) + $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove(); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); retract($entity) end @@ -128,10 +129,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) - $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive()) + $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.remove(); + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); end @@ -140,11 +141,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.addEngines($recommendation.getEngines()); - $recommendation.remove(); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); retract($recommendation); end @@ -153,10 +154,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: RedactionEntity(entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity(entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.remove(); + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); end @@ -165,11 +166,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" salience 32 when - $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) + $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove(); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); retract($lowerRank); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml index 2bf23009..b4895cfb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml @@ -10,7 +10,7 @@ - + diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index c003b9d5..0c1e10e7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -14,6 +14,12 @@ import java.util.Optional; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; @@ -51,197 +57,93 @@ query "getFileAttributes" //------------------------------------ Syngenta specific rules ------------------------------------ -// Rule unit: SYN.0 -rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" +// Rule unit: SYN.1 +rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section(containsString("CTL/") || containsString("BL/")) + $section: Section(containsString("CT") || containsString("BL")) then - Stream.concat( - entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section), - entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) - ).forEach(entity -> { - entity.skip("SYN.0.0", "hint_only"); - entity.addEngine(Engine.RULE); - insert(entity); - }); + /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */ + entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(entity -> entity.skip("SYN.1.0", "")); end //------------------------------------ CBI rules ------------------------------------ -// Rule unit: CBI.3 -rule "CBI.3.0: Redacted because Section contains Vertebrate" +// Rule unit: CBI.0 +rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)" when - $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_author", dictionaryEntry) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRuleAndRedactAndReference( - "CBI.3.0", - "Vertebrate found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - $section.getEntitiesOfType("vertebrate") - ); - }); + $entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end -rule "CBI.3.1: Redacted because Table Row contains Vertebrate" +rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)" when - $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_author", dictionaryEntry) then - $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate")) - .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRuleAndRedactAndReference( - "CBI.3.1", - "Vertebrate found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - $table.getEntitiesOfTypeInSameRow("vertebrate", entity) - ); - }); - end - -rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate" - when - $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) - then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRule("CBI.3.2", "No vertebrate found"); - }); - end - -rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate" - when - $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) - then - $table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate")) - .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRule("CBI.3.3", "No vertebrate found"); - }); + $entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end -// Rule unit: CBI.4 -rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is found in Section" +// Rule unit: CBI.1 +rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_address", dictionaryEntry) + then + $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); + end + +rule "CBI.1.1: Redact CBI Address (Vertebrate Study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_address", dictionaryEntry) + then + $entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +// Rule unit: CBI.2 +rule "CBI.2.0: Don't redact genitive CBI_author" + when + $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied()) + then + entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document) + .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); + end + + +// Rule unit: CBI.7 +rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables" when $section: Section(!hasTables(), - hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("no_redaction_indicator"), + hasEntitiesOfType("published_information"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRuleAndReference( - "CBI.4.0", - "Vertebrate but a no redaction indicator found", - $section.getEntitiesOfType("no_redaction_indicator") + .forEach(redactionEntity -> { + redactionEntity.skipWithReferences( + "CBI.7.0", + "Published Information found in section", + $section.getEntitiesOfType("published_information") ); }); end -rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is found in Table Row" +rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), - hasEntitiesOfType("vertebrate"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $table: Table(hasEntitiesOfType("published_information"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then - $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator")) - .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRuleAndReference( - "CBI.4.1", - "Vertebrate but a no redaction indicator found", - Stream.concat( - $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(), - $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList() - ); - }); - end - - -// Rule unit: CBI.5 -rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in section" - when - $section: Section(!hasTables(), - hasEntitiesOfType("redaction_indicator"), - hasEntitiesOfType("no_redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) - then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRuleAndRedactAndReference( - "CBI.5.0", - "no_redaction_indicator but also redaction_indicator found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - Stream.concat( - $section.getEntitiesOfType("vertebrate").stream(), - $section.getEntitiesOfType("no_redaction_indicator").stream()).toList() - ); - }); - end - -rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Table Row" - when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), - hasEntitiesOfType("redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) - then - $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no-redaction_indicator")) - .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRuleAndRedactAndReference( - "CBI.5.1", - "no_redaction_indicator but also redaction_indicator found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - Stream.concat( - $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(), - $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList() - ); - }); - end - - -// Rule unit: CBI.8 -rule "CBI.8.0: Redacted because Section contains must_redact entity" - when - $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) - then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRuleAndRedactAndReference( - "CBI.8.0", - "must_redact entity found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - $section.getEntitiesOfType("must_redact") - ); - }); - end - -rule "CBI.8.1: Redacted because Table Row contains must_redact entity" - when - $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) - then - $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact")) - .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.addMatchedRuleAndRedactAndReference( - "CBI.8.1", - "must_redact entity found", - "Reg (EC) No 1107/2009 Art. 63 (2g)", - $table.getEntitiesOfTypeInSameRow("must_redact", entity) + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.skipWithReferences( + "CBI.7.1", + "Published Information found in row", + $table.getEntitiesOfTypeInSameRow("published_information", redactionEntity) ); }); end @@ -258,11 +160,7 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.addMatchedRuleAndRedact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)" @@ -275,11 +173,35 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.addMatchedRuleAndRedact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); - insert(redactionEntity); - }); + .forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + + +// Rule unit: CBI.10 +rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)")) + then + $table.streamTableCellsWithHeader("Author(s)") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + +rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author")) + then + $table.streamTableCellsWithHeader("Author") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -294,109 +216,6 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study end -// Rule unit: CBI.12 -rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author" - salience 1 - when - $table: Table(hasHeader("Author(s)") || hasHeader("Author")) - then - Stream.concat( - $table.streamTableCellsWithHeader("Author(s)"), - $table.streamTableCellsWithHeader("Author") - ) - .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.skip("CBI.12.0", "Author(s) header found"); - redactionEntity.addEngine(Engine.RULE); - insert(redactionEntity); - }); - end - -rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No" - when - $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No")) - then - $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No")) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> { - authorEntity.addMatchedRule("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study"); - }); - end - -rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes" - when - $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes")) - then - $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> { - authorEntity.addMatchedRuleAndRedact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); - end - - -// Rule unit: CBI.14 -rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" - when - $sponsorEntity: RedactionEntity(type == "CBI_sponsor", textBefore.contains("batches produced at")) - then - $sponsorEntity.apply("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - end - - -// Rule unit: CBI.15 -rule "CBI.15.0: Redact row if row contains \"determination of residues\" and livestock keyword" - when - $keyword: String() from List.of("livestock", - "live stock", - "tissue", - "tissues", - "liver", - "muscle", - "bovine", - "ruminant", - "ruminants") - $residueKeyword: String() from List.of("determination of residues", "determination of total residues") - $section: Section(!hasTables(), - containsStringIgnoreCase($residueKeyword), - containsStringIgnoreCase($keyword)) - then - entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section) - .forEach(keywordEntity -> insert(keywordEntity)); - - $section.getEntitiesOfType(List.of($keyword, $residueKeyword)) - .forEach(redactionEntity -> { - redactionEntity.addMatchedRuleAndRedact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); - end - -rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword" - when - $keyword: String() from List.of("livestock", - "live stock", - "tissue", - "tissues", - "liver", - "muscle", - "bovine", - "ruminant", - "ruminants") - $residueKeyword: String() from List.of("determination of residues", "determination of total residues") - $table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword)) - then - entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table) - .forEach(keywordEntity -> insert(keywordEntity)); - - $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword)) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(redactionEntity -> { - redactionEntity.addMatchedRuleAndRedact("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - }); - end - - // Rule unit: CBI.16 rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" @@ -407,9 +226,7 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); - insert(entity); }); end @@ -422,8 +239,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); - insert(entity); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); }); end @@ -435,11 +250,7 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) then entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"); - insert(entity); - }); + .forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section")); end rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon" @@ -447,42 +258,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with $section: Section(!hasTables(), containsString("Species:"), containsString("Source:")) then entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"); - insert(entity); - }); - end - - -// Rule unit: CBI.18 -rule "CBI.18.0: Expand CBI_author entities with firstname initials" - no-loop true - when - $entityToExpand: RedactionEntity(type == "CBI_author", - value.matches("[^\\s]+"), - textAfter.startsWith(" "), - anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") - ) - then - RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.removeFromGraph(); - retract($entityToExpand); - insert(expandedEntity); - end - - -// Rule unit: CBI.19 -rule "CBI.19.0: Expand CBI_author entities with salutation prefix" - when - $entityToExpand: RedactionEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) - then - RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - $entityToExpand.removeFromGraph(); - retract($entityToExpand); - insert(expandedEntity); + .forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section")); end @@ -496,9 +272,7 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); - laboratoryEntity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); }); end @@ -511,9 +285,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - laboratoryEntity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); }); end @@ -545,11 +317,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> { - emailEntity.addEngine(Engine.RULE); - emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(emailEntity); - }); + .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" @@ -558,203 +326,45 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> { - emailEntity.addEngine(Engine.RULE); - emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(emailEntity); - }); + .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -// Rule unit: PII.4 -rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" +// Rule unit: PII.2 +rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $contactKeyword: String() from List.of("Contact point:", - "Contact:", - "Alternative contact:", - "European contact:", - "No:", - "Contact:", - "Tel.:", - "Tel:", - "Telephone number:", - "Telephone No:", - "Telephone:", - "Phone No.", - "Phone:", - "Fax number:", - "Fax:", - "E-mail:", - "Email:", - "e-mail:", - "E-mail address:") - $section: Section(containsString($contactKeyword)) + $section: Section(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Ph.") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) then - entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - contactEntity.addEngine(Engine.RULE); - insert(contactEntity); - }); + entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) + .forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)" - when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $contactKeyword: String() from List.of("Contact point:", - "Contact:", - "Alternative contact:", - "European contact:", - "No:", - "Contact:", - "Tel.:", - "Tel:", - "Telephone number:", - "Telephone No:", - "Telephone:", - "Phone No.", - "Phone:", - "Fax number:", - "Fax:", - "E-mail:", - "Email:", - "e-mail:", - "E-mail address:") - $section: Section(containsString($contactKeyword)) - then - entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> { - contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - contactEntity.addEngine(Engine.RULE); - insert(contactEntity); - }); - end - - -// Rule unit: PII.6 -rule "PII.6.0: redact line between contact keywords (non vertebrate study)" - when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) - then - Stream.concat( - entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), - entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) - ) - .forEach(contactEntity -> { - contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - contactEntity.addEngine(Engine.RULE); - insert(contactEntity); - }); - end - -rule "PII.6.1: redact line between contact keywords" +rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + $section: Section(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Ph.") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) then - Stream.concat( - entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), - entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) - ) - .forEach(contactEntity -> { - contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - contactEntity.addEngine(Engine.RULE); - insert(contactEntity); - }); - end - - -// Rule unit: PII.7 -rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" - when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section(getHeadline().containsString("applicant") || - getHeadline().containsString("Primary contact") || - getHeadline().containsString("Alternative contact") || - containsString("Applicant") || - containsString("Telephone number:")) - then - Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", - "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), - Stream.concat( - entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), - entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) - )) - .forEach(entity -> { - entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); - insert(entity); - }); - end - -rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)" - when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section(getHeadline().containsString("applicant") || - getHeadline().containsString("Primary contact") || - getHeadline().containsString("Alternative contact") || - containsString("Applicant") || - containsString("Telephone number:")) - then - Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", - "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), - Stream.concat( - entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), - entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) - )) - .forEach(entity -> { - entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); - insert(entity); - }); - end - - -// Rule unit: PII.8 -rule "PII.8.0: Redact contact information if producer is found" - when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section(containsStringIgnoreCase("producer of the plant protection") || - containsStringIgnoreCase("producer of the active substance") || - containsStringIgnoreCase("manufacturer of the active substance") || - containsStringIgnoreCase("manufacturer:") || - containsStringIgnoreCase("Producer or producers of the active substance")) - then - Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", - "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), - Stream.concat( - entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), - entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) - )) - .forEach(entity -> { - entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - entity.addEngine(Engine.RULE); - insert(entity); - }); - end - -rule "PII.8.1: Redact contact information if producer is found" - when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $section: Section(containsStringIgnoreCase("producer of the plant protection") || - containsStringIgnoreCase("producer of the active substance") || - containsStringIgnoreCase("manufacturer of the active substance") || - containsStringIgnoreCase("manufacturer:") || - containsStringIgnoreCase("Producer or producers of the active substance")) - then - Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:", - "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section), - Stream.concat( - entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), - entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) - )) - .forEach(entity -> { - entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); - insert(entity); - }); + entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) + .forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end @@ -765,11 +375,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)" @@ -778,11 +384,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" @@ -791,11 +393,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); - insert(authorEntity); - }); + .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)" @@ -804,52 +402,20 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) then entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); - insert(authorEntity); - }); - end - - -// Rule unit: PII.11 -rule "PII.11.0: Redact On behalf of Sequani Ltd.:" - when - $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) - then - entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); - end - - -// Rule unit: PII.12 -rule "PII.12.0: Expand PII entities with salutation prefix" - when - $entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) - then - RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"); - expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - expandedEntity.addEngine(Engine.RULE); - insert(expandedEntity); + .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end //------------------------------------ Other rules ------------------------------------ -// Rule unit: ETC.1 -rule "ETC.1.0: Redact Purity" +// Rule unit: ETC.0 +rule "ETC.0.0: Purity Hint" when $section: Section(containsStringIgnoreCase("purity")) then - entityCreationService.byRegex("\\bPurity:\\s*(?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section) - .forEach(entity -> { - entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"); - entity.addEngine(Engine.RULE); - }); - end + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) + .forEach(hint -> hint.skip("ETC.0.0", "hint only")); + end // Rule unit: ETC.2 @@ -858,15 +424,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value == "Yes") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.addMatchedRuleAndRedact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "ETC.2.0: Redact signatures (vertebrate study)" - when - FileAttribute(label == "Vertebrate Study", value == "Yes") - $signature: Image(imageType == ImageType.SIGNATURE) - then - $signature.addMatchedRuleAndRedact("ETC.2.0", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -876,7 +434,7 @@ rule "ETC.3.0: Redact logos (vertebrate study)" not FileAttribute(label == "Vertebrate Study", value == "Yes") $logo: Image(imageType == ImageType.LOGO) then - $logo.addMatchedRuleAndRedact("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end rule "ETC.3.1: Redact logos (non vertebrate study)" @@ -884,16 +442,7 @@ rule "ETC.3.1: Redact logos (non vertebrate study)" FileAttribute(label == "Vertebrate Study", value == "Yes") $logo: Image(imageType == ImageType.LOGO) then - $logo.addMatchedRuleAndRedact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -// Rule unit: ETC.4 -rule "ETC.4.0: Redact dossier dictionary entries" - when - $dossierRedaction: RedactionEntity(type == "dossier_redaction") - then - $dossierRedaction.apply("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -903,54 +452,9 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi not FileAttribute(label == "Confidentiality", value == "confidential") $dossierRedaction: RedactionEntity(type == "dossier_redaction") then - $dossierRedaction.removeFromGraph(); - retract($dossierRedaction); - end - - -// Rule unit: ETC.6 -rule "ETC.6.0: Redact CAS Number" - when - $table: Table(hasHeader("Sample #")) - then - $table.streamTableCellsWithHeader("Sample #") - .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY)) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.addMatchedRuleAndRedact("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"); - insert(redactionEntity); - }); - end - - -// Rule unit: ETC.7 -rule "ETC.7.0: Guidelines FileAttributes" - when - $section: Section(!hasTables(), (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS"))) - then - RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream() - .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) - .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) - .forEach(fileAttribute -> insert(fileAttribute)); - end - - -// Rule unit: ETC.8 -rule "ETC.8.0: Redact formulas (vertebrate study)" - when - not FileAttribute(label == "Vertebrate Study", value == "Yes") - $logo: Image(imageType == ImageType.FORMULA) - then - $logo.addMatchedRuleAndRedact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "ETC.8.1: Redact formulas (non vertebrate study)" - when - FileAttribute(label == "Vertebrate Study", value == "Yes") - $logo: Image(imageType == ImageType.FORMULA) - then - $logo.addMatchedRuleAndRedact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); + update($dossierRedaction); + $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end @@ -963,8 +467,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) - .forEach(entity -> insert(entity)); + .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -975,11 +478,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY")) then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) - .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) - .forEach(entity -> { - entity.addEngine(Engine.NER); - insert(entity); - }); + .map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) + .forEach(entity -> entity.addEngine(Engine.NER)); end @@ -995,6 +495,7 @@ rule "MAN.0.0: Apply manual resize redaction" manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); + $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end @@ -1002,33 +503,44 @@ rule "MAN.0.0: Apply manual resize redaction" rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.setIgnored(true); + $entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction"); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $imageEntityToBeRemoved: Image($id == id) then - $imageEntityToBeRemoved.setIgnored(true); + $imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction"); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); end // Rule unit: MAN.2 rule "MAN.2.0: Apply force redaction" + no-loop true salience 128 when - ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) $entityToForce: RedactionEntity(matchesAnnotationId($id)) then - $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.setRemoved(false); + $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + update($entityToForce); + $entityToForce.getIntersectingNodes().forEach(node -> update(node)); end @@ -1036,10 +548,13 @@ rule "MAN.2.0: Apply force redaction" rule "MAN.3.0: Apply image recategorization" salience 128 when - ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) - $image: Image($id == id) + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) + $imageToBeRecategorized: Image($id == id) then - $image.setImageType(ImageType.fromString($imageType)); + $imageToBeRecategorized.setImageType(ImageType.fromString($imageType)); + update($imageToBeRecategorized); + update($imageToBeRecategorized.getParent()); + retract($recategorization); end @@ -1049,10 +564,10 @@ rule "MAN.3.0: Apply image recategorization" rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when - $larger: RedactionEntity($type: type, $entityType: entityType) - $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + $larger: RedactionEntity($type: type, $entityType: entityType, isActive()) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $contained.removeFromGraph(); + $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); end @@ -1061,15 +576,15 @@ rule "X.0.0: remove Entity contained by Entity of same type" rule "X.1.0: merge intersecting Entities of same type" salience 64 when - $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) - $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) + $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) + $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $first.removeFromGraph(); - $second.removeFromGraph(); RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + $first.remove("X.1.0", "merge intersecting Entities of same type"); + $second.remove("X.1.0", "merge intersecting Entities of same type"); retract($first); retract($second); - insert(mergedEntity); + mergedEntity.getIntersectingNodes().forEach(node -> update(node)); end @@ -1077,10 +592,11 @@ rule "X.1.0: merge intersecting Entities of same type" rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) - $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive()) + $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $entity.removeFromGraph(); + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); retract($entity) end @@ -1089,10 +605,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) - $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive()) + $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.removeFromGraph(); + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); end @@ -1101,11 +617,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then $entity.addEngines($recommendation.getEngines()); - $recommendation.removeFromGraph(); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); retract($recommendation); end @@ -1114,10 +630,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" salience 256 when - $entity: RedactionEntity(entityType == EntityType.ENTITY) - $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: RedactionEntity(entityType == EntityType.ENTITY, isActive()) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $recommendation.removeFromGraph(); + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); retract($recommendation); end @@ -1126,10 +642,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" salience 32 when - $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY) - $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) + $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive()) + $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive()) then - $lowerRank.removeFromGraph(); + $lowerRank.getIntersectingNodes().forEach(node -> update(node)); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); retract($lowerRank); end @@ -1156,9 +673,5 @@ rule "LDS.0.0: run local dictionary search" when DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then - entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - insert(entity); - }); + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document).toList(); end