diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java index 275f7e83..99511a8f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java @@ -1,6 +1,6 @@ package com.iqser.red.service.redaction.v1.server.model; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import lombok.Builder; import lombok.Getter; @@ -12,6 +12,6 @@ import lombok.Setter; public class ClosestEntity { private double distance; - private TextRange textRange; + private TextEntity textEntity; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java index 5b47e232..d33f588e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java @@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.model; import java.util.List; import java.util.PriorityQueue; +import java.util.UUID; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; @@ -69,6 +71,7 @@ public class ManualEntity implements IEntity { .build(); } + public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) { List rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList(); @@ -90,6 +93,19 @@ public class ManualEntity implements IEntity { } + public static ManualEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) { + + List rectangleWithPages = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList(); + return ManualEntity.builder() + .id(UUID.randomUUID().toString()) + .value(manualResizeRedaction.getValue()) + .entityPosition(rectangleWithPages) + .entityType(EntityType.ENTITY) + .manualOverwrite(new ManualChangeOverwrite(EntityType.ENTITY)) + .build(); + } + + @Override public TextRange getTextRange() { @@ -103,6 +119,7 @@ public class ManualEntity implements IEntity { return getManualOverwrite().getType().orElse(type); } + private static EntityType getEntityType(EntryType entryType) { switch (entryType) { @@ -124,7 +141,9 @@ public class ManualEntity implements IEntity { } } + private EntityType getEntityType(boolean isHint) { + return isHint ? EntityType.HINT : EntityType.ENTITY; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java index 48c3beea..dd208f56 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java @@ -307,7 +307,7 @@ public final class MigrationEntity { .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) - .type(entity.getType()) + .type(entity.type()) .section(redactionLogEntry.getSection()) .textAfter(redactionLogEntry.getTextAfter()) .textBefore(redactionLogEntry.getTextBefore()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index 6cd61fca..fad8c114 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -128,19 +128,19 @@ public class Dictionary { public void recommendEverywhere(TextEntity textEntity) { - addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), false); + addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false); } public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) { - addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), true); + addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true); } public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) { - splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.getType(), authorName, textEntity.getMatchedRuleList(), true)); + splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index a0a0c5c9..a649889d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity; import java.awt.geom.Rectangle2D; +import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.HashSet; @@ -33,13 +34,15 @@ public class TextEntity implements IEntity { // primary key @EqualsAndHashCode.Include - final TextRange textRange; - @EqualsAndHashCode.Include - final String type; - @EqualsAndHashCode.Include - final EntityType entityType; + final String id; // primary key end + TextRange textRange; + @Builder.Default + List duplicateTextRanges = new ArrayList<>(); + String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted + final EntityType entityType; + @Builder.Default final PriorityQueue matchedRuleList = new PriorityQueue<>(); final ManualChangeOverwrite manualOverwrite; @@ -62,12 +65,30 @@ public class TextEntity implements IEntity { SemanticNode deepestFullyContainingNode; - public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType) { + public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) { - return TextEntity.builder().type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build(); + return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build(); } + public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, String id) { + + return TextEntity.builder().id(id).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build(); + } + + + private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) { + + Map> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange); + return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name()); + } + + + public void addTextRange(TextRange textRange) { + + duplicateTextRanges.add(textRange); + } + public boolean occursInNodeOfType(Class clazz) { return intersectingNodes.stream().anyMatch(clazz::isInstance); @@ -82,13 +103,13 @@ public class TextEntity implements IEntity { public boolean isType(String type) { - return this.type.equals(type); + return type().equals(type); } public boolean isAnyType(List types) { - return types.contains(type); + return types.contains(type()); } @@ -125,7 +146,6 @@ public class TextEntity implements IEntity { .min(Comparator.comparingInt(Page::getNumber)) .orElseThrow(() -> new RuntimeException("No Positions found on any page!")); - String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name()); positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList(); } return positionsOnPagePerPage; @@ -193,7 +213,7 @@ public class TextEntity implements IEntity { }); sb.delete(sb.length() - 2, sb.length()); sb.append("], type = \""); - sb.append(type); + sb.append(type()); sb.append("\", EntityType."); sb.append(entityType); sb.append("]"); @@ -207,4 +227,11 @@ public class TextEntity implements IEntity { return getManualOverwrite().getType().orElse(type); } + + @Override + public String value() { + + return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java index 4201f353..79484363 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java @@ -19,6 +19,7 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; @@ -27,8 +28,12 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Document implements GenericSemanticNode { + @EqualsAndHashCode.Include + List treeId = Collections.emptyList(); + Set pages; DocumentTree documentTree; Integer numberOfPages; @@ -67,13 +72,6 @@ public class Document implements GenericSemanticNode { } - @Override - public List getTreeId() { - - return Collections.emptyList(); - } - - @Override public void setTreeId(List tocId) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java index 0d71164f..c61bff64 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java @@ -23,22 +23,21 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Footer implements GenericSemanticNode { - @Builder.Default final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty(); + @EqualsAndHashCode.Include List treeId; + TextBlock leafTextBlock; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @Builder.Default - @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); - @EqualsAndHashCode.Exclude Map bBoxCache; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java index 0575e37d..270a4960 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java @@ -23,22 +23,21 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Header implements GenericSemanticNode { - @Builder.Default final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty(); + @EqualsAndHashCode.Include List treeId; + TextBlock leafTextBlock; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @Builder.Default - @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); - @EqualsAndHashCode.Exclude Map bBoxCache; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java index 31006ddf..a2c748e9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java @@ -24,20 +24,19 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Headline implements GenericSemanticNode { + @EqualsAndHashCode.Include List treeId; TextBlock leafTextBlock; SectionIdentifier sectionIdentifier; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @Builder.Default - @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); - @EqualsAndHashCode.Exclude Map bBoxCache; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java index 2f0c8a2e..e591af11 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java @@ -32,8 +32,10 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Image implements GenericSemanticNode, IEntity { + @EqualsAndHashCode.Include List treeId; String id; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java index 18aee858..f741bf62 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java @@ -23,26 +23,23 @@ import lombok.experimental.FieldDefaults; @NoArgsConstructor @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Page { + @EqualsAndHashCode.Include Integer number; Integer height; Integer width; Integer rotation; - @EqualsAndHashCode.Exclude List mainBody; - @EqualsAndHashCode.Exclude Header header; - @EqualsAndHashCode.Exclude Footer footer; @Builder.Default - @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); @Builder.Default - @EqualsAndHashCode.Exclude Set images = new HashSet<>(); @@ -57,19 +54,4 @@ public class Page { return String.valueOf(number); } - - - @Override - public int hashCode() { - - return number; - } - - - @Override - public boolean equals(Object o) { - - return o instanceof Page && o.hashCode() == this.hashCode(); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java index 4d252731..db5c38aa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java @@ -21,19 +21,18 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Paragraph implements GenericSemanticNode { + @EqualsAndHashCode.Include List treeId; TextBlock leafTextBlock; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @Builder.Default - @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); - @EqualsAndHashCode.Exclude Map bBoxCache; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java index 06c7cbe2..58259feb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java @@ -24,19 +24,17 @@ import lombok.extern.slf4j.Slf4j; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Section implements GenericSemanticNode { - + @EqualsAndHashCode.Include List treeId; TextBlock textBlock; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @Builder.Default - @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); - @EqualsAndHashCode.Exclude Map bBoxCache; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java index af423530..d874ea24 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java @@ -19,6 +19,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; @@ -80,8 +81,8 @@ public interface SemanticNode { */ default Set getPages(TextRange textRange) { - if (!getTextRange().contains(textRange)) { - throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", textRange, getTextRange())); + if (!getTextRange().intersects(textRange)) { + throw new IllegalArgumentException(format("%s which was used to query for pages is not intersected in the %s of this node!", textRange, getTextRange())); } return getTextBlock().getPages(textRange); } @@ -247,7 +248,7 @@ public interface SemanticNode { */ default boolean hasEntitiesOfType(String type) { - return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.getType().equals(type)); + return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type)); } @@ -260,7 +261,7 @@ public interface SemanticNode { */ default boolean hasEntitiesOfAnyType(String... types) { - return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type))); + return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type))); } @@ -273,7 +274,7 @@ public interface SemanticNode { */ default boolean hasEntitiesOfAllTypes(String... types) { - return getEntities().stream().filter(TextEntity::active).map(TextEntity::getType).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList()); + return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList()); } @@ -286,7 +287,7 @@ public interface SemanticNode { */ default List getEntitiesOfType(String type) { - return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList(); + return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList(); } @@ -627,6 +628,27 @@ public interface SemanticNode { } + /** + * For a given TextRange this function returns a List of rectangle around the text in the range. + * These Rectangles are split either by a new line or by a large gap in the current line. + * This is mainly used to find the positions of TextEntities + * + * @param textRange A TextRange to calculate the positions for. + * @return A Map, where the keys are the pages and the values are a list of rectangles describing the position of words + */ + default Map> getPositionsPerPage(TextRange textRange) { + + if (isLeaf()) { + return getTextBlock().getPositionsPerPage(textRange); + } + Optional containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst(); + if (containingChildNode.isEmpty()) { + return getTextBlock().getPositionsPerPage(textRange); + } + return containingChildNode.get().getPositionsPerPage(textRange); + } + + /** * If this Node is a Leaf it will calculate the boundingBox of its LeafTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children. * If called on the Document, it will return the cropbox of each page @@ -693,4 +715,18 @@ public interface SemanticNode { return bBoxPerPage; } + + /** + * Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node. + * The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal. + * + * @param visitor The {@link NodeVisitor} to accept and apply during the traversal. + * @see NodeVisitor + */ + default void accept(NodeVisitor visitor) { + + visitor.visit(this); + streamChildren().forEach(childNode -> childNode.accept(visitor)); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java index 09320581..100f3fe9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java @@ -28,8 +28,10 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Table implements SemanticNode { + @EqualsAndHashCode.Include List treeId; DocumentTree documentTree; @@ -39,10 +41,8 @@ public class Table implements SemanticNode { TextBlock textBlock; @Builder.Default - @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); - @EqualsAndHashCode.Exclude Map bBoxCache; @@ -128,7 +128,7 @@ public class Table implements SemanticNode { List rowsWithEntityOfType = getEntities().stream() .filter(TextEntity::active) - .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType()))) + .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.type()))) .map(TextEntity::getIntersectingNodes) .filter(node -> node instanceof TableCell) .map(node -> (TableCell) node) @@ -153,7 +153,7 @@ public class Table implements SemanticNode { .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities) .flatMap(Collection::stream) .filter(TextEntity::active) - .noneMatch(entity -> types.contains(entity.getType()))) + .noneMatch(entity -> types.contains(entity.type()))) .flatMap(this::streamRow) .map(TableCell::getEntities) .flatMap(Collection::stream); @@ -196,7 +196,7 @@ public class Table implements SemanticNode { */ public Stream streamTableCellsWhichContainType(String type) { - return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.getType().equals(type))); + return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.type().equals(type))); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java index 6165c618..24658449 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java @@ -23,8 +23,10 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode(onlyExplicitlyIncluded = true) public class TableCell implements GenericSemanticNode { + @EqualsAndHashCode.Include List treeId; int row; int col; @@ -36,11 +38,9 @@ public class TableCell implements GenericSemanticNode { TextBlock textBlock; - @EqualsAndHashCode.Exclude DocumentTree documentTree; @Builder.Default - @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java index 57334d5f..9f372ed4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java @@ -71,7 +71,16 @@ public class ConcatenatedTextBlock implements TextBlock { private List getAllAtomicTextBlocksPartiallyInStringBoundary(TextRange textRange) { - return atomicTextBlocks.stream().filter(tb -> tb.getTextRange().intersects(textRange)).toList(); + List intersectingAtomicTextBlocks = new LinkedList<>(); + for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) { + if (atomicTextBlock.getTextRange().start() > textRange.end()) { + break; // early stop, following TextBlocks will never intersect + } + if (atomicTextBlock.getTextRange().intersects(textRange)) { + intersectingAtomicTextBlocks.add(atomicTextBlock); + } + } + return intersectingAtomicTextBlocks; } @@ -122,13 +131,13 @@ public class ConcatenatedTextBlock implements TextBlock { } - @Override public Rectangle2D getPosition(int stringIdx) { return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx); } + public TextRange getLineTextRange(int lineNumber) { if (atomicTextBlocks.size() == 1) { @@ -144,10 +153,10 @@ public class ConcatenatedTextBlock implements TextBlock { return new TextRange(textRange.start(), textRange.start()); } + @Override public List getPositions(TextRange stringTextRange) { - List textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange); if (textBlocks.isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index d0271272..5cf13382 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -142,44 +142,39 @@ public class EntityLogCreatorService { private List createEntityLogEntries(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { List entries = new ArrayList<>(); - Set processedIds = new HashSet<>(); document.getEntities() .stream() .filter(entity -> !entity.getValue().isEmpty()) .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation) .filter(entity -> !entity.removed()) - .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds))); + .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode))); document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); - notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId))); + notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity))); return entries; } - private List toEntityLogEntries(TextEntity textEntity, String dossierTemplateId, Set processedIds) { + private List toEntityLogEntries(TextEntity textEntity) { - List redactionLogEntities = new ArrayList<>(); + List entityLogEntries = new ArrayList<>(); + // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { - // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows. - if (processedIds.contains(positionOnPage.getId())) { - continue; - } - processedIds.add(positionOnPage.getId()); - - EntityLogEntry entityLogEntries = createEntityLogEntry(textEntity, dossierTemplateId); - entityLogEntries.setId(positionOnPage.getId()); + EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity); List rectanglesPerLine = positionOnPage.getRectanglePerLine() .stream() .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) .toList(); - entityLogEntries.setPositions(rectanglesPerLine); - redactionLogEntities.add(entityLogEntries); + // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages + entityLogEntry.setId(positionOnPage.getId()); + entityLogEntry.setPositions(rectanglesPerLine); + entityLogEntries.add(entityLogEntry); } - return redactionLogEntities; + return entityLogEntries; } @@ -190,8 +185,6 @@ public class EntityLogCreatorService { return EntityLogEntry.builder() .id(image.getId()) .value(image.value()) - .color(getColor(imageType, dossierTemplateId, image.applied(), isHint)) - .value(image.value()) .type(imageType) .reason(image.buildReasonWithManualChangeDescriptions()) .legalBasis(image.legalBasis()) @@ -210,13 +203,12 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) { + private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) { String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); boolean isHint = isHint(manualEntity.getEntityType()); return EntityLogEntry.builder() .id(manualEntity.getId()) - .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint)) .reason(manualEntity.buildReasonWithManualChangeDescriptions()) .legalBasis(manualEntity.legalBasis()) .value(manualEntity.value()) @@ -246,17 +238,16 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) { + private EntityLogEntry createEntityLogEntry(TextEntity entity) { Set referenceIds = new HashSet<>(); entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); boolean isHint = isHint(entity.getEntityType()); return EntityLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) - .type(entity.getType()) + .type(entity.type()) .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString())) .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId()) .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) @@ -282,16 +273,6 @@ public class EntityLogCreatorService { } - private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) { - - if (!isApplied && !isHint) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - - } - - private EntryState buildEntryState(IEntity entity) { if (entity.applied() && entity.active()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index 55bf7620..c0f9c402 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -1,54 +1,70 @@ package com.iqser.red.service.redaction.v1.server.service; import java.awt.geom.Rectangle2D; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import org.springframework.stereotype.Service; + +import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.redaction.v1.server.model.ClosestEntity; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; -import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; +@Service @RequiredArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ManualChangesApplicationService { - EntityCreationService entityCreationService; + static double MATCH_THRESHOLD = 10; - public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) { + EntityFindingUtility entityFindingUtility; - if (IEntityToBeReCategorized instanceof Image image) { + + @Deprecated + public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) { + + entityToBeReCategorized.getMatchedRuleList().clear(); + entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization); + + if (entityToBeReCategorized instanceof Image image) { image.setImageType(ImageType.fromString(manualRecategorization.getType())); return; } - // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed! - if (IEntityToBeReCategorized instanceof TextEntity textEntity) { - TextEntity recategorizedEntity = entityCreationService.copyEntityWithoutRules(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode()); - recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage()); - recategorizedEntity.getManualOverwrite().addChange(manualRecategorization); - textEntity.remove("FINAL.0.0", "removed by manual recategorization"); + + if (entityToBeReCategorized instanceof TextEntity textEntity) { + textEntity.setType(manualRecategorization.getType()); } } + public void resize(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) { + + resizeEntityAndReinsert(entityToBeResized, manualResizeRedaction); + } + + + @Deprecated public void resizeEntityAndReinsert(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) { PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage() @@ -57,96 +73,62 @@ public class ManualChangesApplicationService { .findFirst() .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!")); - positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList())); + positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions() + .stream() + .map(ManualChangesApplicationService::toRectangle2D) + .collect(Collectors.toList())); - String value = manualResizeRedaction.getValue(); - int newStartOffset = -1; SemanticNode node = entityToBeResized.getDeepestFullyContainingNode(); - ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build(); - + ManualEntity searchEntity = ManualEntity.fromManualResizeRedaction(manualResizeRedaction); // Loop through nodes starting from the deepest fully containing node all the way to the document node while (node != null) { - if (node.containsString(value)) { - SearchImplementation searchImplementation = new SearchImplementation(value, false); - List textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()); - for (TextRange textRange : textRanges) { - SemanticNode finalNode = node; + Map> possibleEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, List.of(searchEntity)); + Optional closestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(searchEntity, possibleEntities, MATCH_THRESHOLD); - List tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange) - .stream() - .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode)) - .collect(Collectors.toList()); - - // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one. - determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity); - - // Remove all temp entities from the graph - tempEntities.forEach(TextEntity::removeFromGraph); - } - break; + if (closestEntity.isPresent()) { + copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get()); + possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); + return; } - // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere. + possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); + if (node.hasParent()) { node = node.getParent(); } else { break; } } + } - if (closestEntity.getTextRange() != null) { - newStartOffset = closestEntity.getTextRange().start(); - } - // need to reinsert the entity, due to the boundary having changed. - if (newStartOffset > -1) { - removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset); - } + private static void copyValuesFromClosestEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, TextEntity closestEntity) { + + Set currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes()); + Set newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes()); + + Sets.difference(currentIntersectingNodes, newIntersectingNodes).forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized)); + Sets.difference(newIntersectingNodes, currentIntersectingNodes).forEach(addedNode -> addedNode.getEntities().add(entityToBeResized)); + + Set currentIntersectingPages = new HashSet<>(entityToBeResized.getPages()); + Set newIntersectingPages = new HashSet<>(closestEntity.getPages()); + + Sets.difference(currentIntersectingPages, newIntersectingPages).forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized)); + Sets.difference(newIntersectingPages, currentIntersectingPages).forEach(addedPage -> addedPage.getEntities().add(entityToBeResized)); + + entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); + entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes)); + entityToBeResized.setTextRange(closestEntity.getTextRange()); + entityToBeResized.setTextAfter(closestEntity.getTextAfter()); + entityToBeResized.setTextBefore(closestEntity.getTextBefore()); + entityToBeResized.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges())); + entityToBeResized.setValue(closestEntity.getValue()); + entityToBeResized.setPages(newIntersectingPages); entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction); } - public static void determineCorrectEntity(ManualResizeRedaction manualResizeRedaction, - TextRange textRange, - List tempEntities, - ClosestEntity closestEntity) { - - double currentDistance = calculateClosest(manualResizeRedaction.getPositions().get(0), tempEntities.get(0).getPositionsOnPagePerPage()); - if (currentDistance < closestEntity.getDistance()) { - closestEntity.setDistance(currentDistance); - closestEntity.setTextRange(textRange); - } - } - - - private static double calculateClosest(Rectangle position, List positionOnPages) { - - Rectangle2D rectangle2D = positionOnPages.get(0).getRectanglePerLine().get(0); - double difference = 0; - difference += position.getTopLeftX() > rectangle2D.getX() ? (position.getTopLeftX() - rectangle2D.getX()) : (rectangle2D.getX() - position.getTopLeftX()); - difference += position.getTopLeftY() > rectangle2D.getY() ? (position.getTopLeftY() - rectangle2D.getY()) : (rectangle2D.getY() - position.getTopLeftY()); - difference += position.getWidth() > rectangle2D.getWidth() ? (position.getWidth() - rectangle2D.getWidth()) : (rectangle2D.getWidth() - position.getWidth()); - difference += position.getHeight() > rectangle2D.getHeight() ? (position.getHeight() - rectangle2D.getHeight()) : (rectangle2D.getHeight() - position.getHeight()); - return difference; - } - - - private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) { - - SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode(); - entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this)); - entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this)); - entityToBeResized.setIntersectingNodes(new LinkedList<>()); - entityToBeResized.setDeepestFullyContainingNode(null); - entityToBeResized.setPages(new HashSet<>()); - entityToBeResized.getTextRange().setStart(newStartOffset); - entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length()); - - entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto); - } - - public void resizeImage(Image image, ManualResizeRedaction manualResizeRedaction) { if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index 968e16c4..596167ea 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -100,11 +100,11 @@ public class RedactionLogCreatorService { int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); boolean isHint = isHint(entity.getEntityType()); return RedactionLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint)) + .color(getColor(entity.type(), dossierTemplateId, entity.applied(), isHint)) .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) - .type(entity.getType()) + .type(entity.type()) .redacted(entity.applied()) .isHint(isHint) .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION)) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java index d9aa03ea..dc1c49aa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java @@ -3,10 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -23,11 +21,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.redaction.v1.model.AnalyzeResponse; import com.iqser.red.service.redaction.v1.model.QueueNames; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; -import com.iqser.red.service.redaction.v1.server.model.ClosestEntity; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; -import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; @@ -62,6 +57,8 @@ public class UnprocessedChangesService { final EntityEnrichmentService entityEnrichmentService; final ManualEntityCreationService manualEntityCreationService; final DictionaryService dictionaryService; + final ManualChangesApplicationService manualChangesApplicationService; + EntityCreationService entityCreationService; @@ -145,8 +142,21 @@ public class UnprocessedChangesService { continue; } - TextEntity correctEntity = createCorrectEntity(manualEntity, document, optionalTextEntity.get().getTextRange()); - resizeEntityAndReinsert(correctEntity, manualResizeRedactions.stream().filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())).findFirst()); + TextEntity correctEntity = createCorrectEntity(manualEntity, optionalTextEntity.get()); + Optional optionalManualResizeRedaction = manualResizeRedactions.stream() + .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())) + .findFirst(); + if (optionalManualResizeRedaction.isPresent()) { + ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get(); + manualChangesApplicationService.resizeEntityAndReinsert(correctEntity, manualResizeRedaction); + + // If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it + // from the graph, so it does not get processed and sent back to persistence-service to update its value. + if (!correctEntity.getValue().equals(manualResizeRedaction.getValue())) { + correctEntity.removeFromGraph(); + } + } + } // remove all temp entities from the graph @@ -154,101 +164,27 @@ public class UnprocessedChangesService { } - public void resizeEntityAndReinsert(TextEntity entityToBeResized, Optional optionalManualResizeRedaction) { + private TextEntity createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) { - if (optionalManualResizeRedaction.isEmpty()) { - return; - } + TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId()); - ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get(); + correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); + correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); + correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges())); + correctEntity.setPages(new HashSet<>(closestEntity.getPages())); - PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage() - .stream() - .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId())) - .findFirst() - .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!")); + correctEntity.setValue(closestEntity.getValue()); + correctEntity.setTextAfter(closestEntity.getTextAfter()); + correctEntity.setTextBefore(closestEntity.getTextBefore()); - positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList())); - - String value = manualResizeRedaction.getValue(); - int newStartOffset = -1; - SemanticNode node = entityToBeResized.getDeepestFullyContainingNode(); - ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build(); - - // Loop through nodes starting from the deepest fully containing node all the way to the document node - while (node != null) { - if (node.containsString(value)) { - SearchImplementation searchImplementation = new SearchImplementation(value, false); - List textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()); - - for (TextRange textRange : textRanges) { - SemanticNode finalNode = node; - - List tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange) - .stream() - .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode)) - .collect(Collectors.toList()); - - // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one. - ManualChangesApplicationService.determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity); - - // Remove all temp entities from the graph - tempEntities.forEach(TextEntity::removeFromGraph); - } - break; - } - - // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere. - if (node.hasParent()) { - node = node.getParent(); - } else { - break; - } - } - - if (closestEntity.getTextRange() != null) { - newStartOffset = closestEntity.getTextRange().start(); - } - - // need to reinsert the entity, due to the boundary having changed. - removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset); - entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction); - } - - - private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) { - - SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode(); - entityToBeResized.removeFromGraph(); - entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this)); - entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this)); - entityToBeResized.setIntersectingNodes(new LinkedList<>()); - entityToBeResized.setDeepestFullyContainingNode(null); - entityToBeResized.setPages(new HashSet<>()); - entityToBeResized.getTextRange().setStart(newStartOffset); - entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length()); - // Don't insert into the graph if newStartOffset is -1 because it means nothing was found. - if (newStartOffset > -1) { - entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto); - } - } - - - private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) { - - TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node); + correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); + correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); - List redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size()); - for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) { - redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine())); - } - correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage); - return correctEntity; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index bdfa11d5..4a9d31d4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.service.document; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*; import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import java.util.Collection; @@ -18,24 +19,21 @@ import org.kie.api.runtime.KieSession; import com.google.common.base.Functions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; -import com.iqser.red.service.redaction.v1.server.model.NerEntities; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; import lombok.RequiredArgsConstructor; @@ -203,6 +201,7 @@ public class EntityCreationService { return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node); } + public Stream shortestBetweenAnyStringIgnoreCase(List starts, List stops, String type, EntityType entityType, SemanticNode node, int limit) { checkIfBothStartAndEndAreEmpty(starts, stops); @@ -233,9 +232,10 @@ public class EntityCreationService { return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node); } + public Stream betweenTextRanges(List startBoundaries, List stopBoundaries, String type, EntityType entityType, SemanticNode node) { - return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node,0); + return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node, 0); } @@ -282,22 +282,11 @@ public class EntityCreationService { } - private void checkIfBothStartAndEndAreEmpty(String start, String end) { - checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end)); - } - - - private void checkIfBothStartAndEndAreEmpty(List start, List end) { - if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) { - throw new IllegalArgumentException("Start and end values are empty!"); - } - } - - public Stream bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) { return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) - .stream().filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary)) + .stream() + .filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary)) .map(bounds -> byTextRange(bounds, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -309,7 +298,9 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); SearchImplementation searchImplementation = new SearchImplementation(strings, false); return searchImplementation.getBoundaries(textBlock, node.getTextRange()) - .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary)) + .stream() + .map(boundary -> toLineAfterTextRange(textBlock, boundary)) + .filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -321,7 +312,9 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); SearchImplementation searchImplementation = new SearchImplementation(strings, true); return searchImplementation.getBoundaries(textBlock, node.getTextRange()) - .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary)) + .stream() + .map(boundary -> toLineAfterTextRange(textBlock, boundary)) + .filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -332,7 +325,9 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); return RedactionSearchUtility.findTextRangesByString(string, textBlock) - .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary)) + .stream() + .map(boundary -> toLineAfterTextRange(textBlock, boundary)) + .filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -343,7 +338,9 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); return RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, textBlock) - .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary)) + .stream() + .map(boundary -> toLineAfterTextRange(textBlock, boundary)) + .filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) .map(Optional::get); @@ -352,7 +349,8 @@ public class EntityCreationService { public Stream lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) { - return tableNode.streamTableCells().flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()), + return tableNode.streamTableCells() + .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()), tableCell, type, entityType, @@ -545,7 +543,7 @@ public class EntityCreationService { public Optional byPrefixExpansionRegex(TextEntity entity, String regexPattern) { int expandedStart = RedactionSearchUtility.getExpandedStartByRegex(entity, regexPattern); - return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); + return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); } @@ -553,18 +551,10 @@ public class EntityCreationService { int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern); expandedEnd = truncateEndIfLineBreakIsBetween(entity.getTextRange().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock()); - return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); + return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); } - private int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) { - - if (textBlock.getNextLinebreak(end) < expandedEnd) { - return end; - } - return expandedEnd; - } - /** * Creates a redaction entity based on the given boundary, type, entity type, and semantic node. * If the document already contains an equal redaction entity, then the original Entity is returned. @@ -581,6 +571,7 @@ public class EntityCreationService { return byTextRangeWithEngine(textRange, type, entityType, node, Set.of(Engine.RULE)); } + /** * Creates a redaction entity based on the given boundary, type, entity type, and semantic node. * If the document already contains an equal redaction entity, then the original Entity is returned. @@ -599,9 +590,21 @@ public class EntityCreationService { throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node)); } TextRange trimmedTextRange = textRange.trim(node.getTextBlock()); - TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType); + TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node); if (node.getEntities().contains(entity)) { - return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngines(engines)).findAny(); + Optional optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny(); + if (optionalTextEntity.isEmpty()) { + return optionalTextEntity; // Entity has been recategorized and should not be created at all. + } + TextEntity existingEntity = optionalTextEntity.get(); + if (existingEntity.getTextRange().equals(textRange)) { + return optionalTextEntity; // exactly the same entity, return directly + } + if (!existingEntity.resized()) { + addDuplicateEntityToGraph(existingEntity, textRange, node); + return optionalTextEntity; // If Entity has not been resized, insert as duplicate at appropriate position + } + return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there } addEntityToGraph(entity, node); entity.addEngines(engines); @@ -610,10 +613,23 @@ public class EntityCreationService { } + // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here. + @Deprecated(forRemoval = true) public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) { TextRange trimmedTextRange = textRange.trim(node.getTextBlock()); - TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType); + TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node); + addEntityToGraph(entity, node); + return entity; + } + + + // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here. + @Deprecated(forRemoval = true) + public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node, String id) { + + TextRange trimmedTextRange = textRange.trim(node.getTextBlock()); + TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, id); addEntityToGraph(entity, node); return entity; } @@ -631,7 +647,7 @@ public class EntityCreationService { return entitiesToMerge.get(0); } - TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType); + TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType, node); mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet())); entitiesToMerge.stream().map(TextEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule)); entitiesToMerge.stream() @@ -662,6 +678,7 @@ public class EntityCreationService { return newEntity; } + public TextEntity copyEntityWithoutRules(TextEntity entity, String type, EntityType entityType, SemanticNode node) { TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException( @@ -683,7 +700,8 @@ public class EntityCreationService { public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!")); + return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException( + "No entity present!")); } @@ -693,6 +711,18 @@ public class EntityCreationService { } + public Optional optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { + + return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)); + } + + + public Optional optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { + + return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)); + } + + public Stream combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) { return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) @@ -702,21 +732,6 @@ public class EntityCreationService { } - public TextEntity byTableCellAsHighlight(TableCell tableCell, String type, EntityType entityType) { - - TextEntity highlightEntity = TextEntity.initialEntityNode(new TextRange(tableCell.getTextRange().start(), tableCell.getTextRange().start()), type, entityType); - - String positionId = IdBuilder.buildId(tableCell.getBBox().keySet(), tableCell.getBBox().values().stream().toList(), type, entityType.name()); - highlightEntity.setPositionsOnPagePerPage(tableCell.getBBox() - .entrySet() - .stream() - .map(entry -> new PositionOnPage(positionId, entry.getKey(), List.of(entry.getValue()))) - .toList()); - addEntityToGraph(highlightEntity, tableCell); - return highlightEntity; - } - - public boolean isValidEntityTextRange(TextBlock textBlock, TextRange textRange) { return textRange.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, textRange); @@ -727,9 +742,17 @@ public class EntityCreationService { DocumentTree documentTree = node.getDocumentTree(); try { - addEntityToGraph(entity, documentTree); - entity.addIntersectingNode(documentTree.getRoot().getNode()); - documentTree.getRoot().getNode().getEntities().add(entity); + if (node.getEntities().contains(entity)) { + // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges + node.getEntities().stream()// + .filter(e -> e.equals(entity))// + .filter(e -> !e.getTextRange().equals(entity.getTextRange()))// + .findAny()// + .ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node)); + } else { + entity.addIntersectingNode(documentTree.getRoot().getNode()); + addEntityToGraph(entity, documentTree); + } } catch (NoSuchElementException e) { entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode()); entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock()); @@ -740,6 +763,33 @@ public class EntityCreationService { } + private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) { + + entityToDuplicate.addTextRange(newTextRange); + + SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes() + .stream() + .sorted(Comparator.comparingInt(n -> -n.getTreeId().size())) + .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges().stream().allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && // + intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange())) + .findFirst() + .orElse(node.getDocumentTree().getRoot().getNode()); + + entityToDuplicate.setDeepestFullyContainingNode(deepestSharedNode); + + Set additionalIntersectingNodes = findIntersectingSubNodes(deepestSharedNode, newTextRange); + + additionalIntersectingNodes.forEach(additionalIntersectingNode -> { + if (entityToDuplicate.getIntersectingNodes().contains(additionalIntersectingNode)) { + return; + } + additionalIntersectingNode.getEntities().add(entityToDuplicate); + additionalIntersectingNode.getPages(newTextRange).forEach(page -> page.getEntities().add(entityToDuplicate)); + entityToDuplicate.addIntersectingNode(additionalIntersectingNode); + }); + } + + private void addEntityToGraph(TextEntity entity, DocumentTree documentTree) { SemanticNode containingNode = documentTree.childNodes(Collections.emptyList()) @@ -757,45 +807,4 @@ public class EntityCreationService { } - private static void addToPages(TextEntity entity) { - - Set pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange()); - entity.getPages().addAll(pages); - pages.forEach(page -> page.getEntities().add(entity)); - } - - - private static void addEntityToNodeEntitySets(TextEntity entity) { - - entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity)); - } - - - private static boolean allEntitiesIntersectAndHaveSameTypes(List entitiesToMerge) { - - if (entitiesToMerge.isEmpty()) { - return true; - } - TextEntity previousEntity = entitiesToMerge.get(0); - for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) { - boolean typeMatches = textEntity.getType().equals(previousEntity.getType()); - boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType()); - boolean intersects = textEntity.intersects(previousEntity); - if (!typeMatches || !entityTypeMatches || !intersects) { - return false; - } - } - return true; - } - - - private static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) { - - if (textBlock.getTextRange().end() == textRange.end()) { - return new TextRange(textRange.end(), textRange.end()); - } - - return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java new file mode 100644 index 00000000..25f1b0d1 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java @@ -0,0 +1,91 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import java.util.List; +import java.util.Set; + +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; + +public class EntityCreationUtility { + + + public static void checkIfBothStartAndEndAreEmpty(String start, String end) { + + checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end)); + } + + + public static void checkIfBothStartAndEndAreEmpty(List start, List end) { + + if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) { + throw new IllegalArgumentException("Start and end values are empty!"); + } + } + + + public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) { + + if (textBlock.getNextLinebreak(end) < expandedEnd) { + return end; + } + return expandedEnd; + } + + + public static Set findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) { + + IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange); + + if (initialIntersectingNode.getTextRange().intersects(textRange)) { + initialIntersectingNode.accept(visitor); + } + + return visitor.getIntersectingNodes(); + } + + + public static void addToPages(TextEntity entity) { + + Set pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange()); + entity.getPages().addAll(pages); + pages.forEach(page -> page.getEntities().add(entity)); + } + + + public static void addEntityToNodeEntitySets(TextEntity entity) { + + entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity)); + } + + + public static boolean allEntitiesIntersectAndHaveSameTypes(List entitiesToMerge) { + + if (entitiesToMerge.isEmpty()) { + return true; + } + TextEntity previousEntity = entitiesToMerge.get(0); + for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) { + boolean typeMatches = textEntity.type().equals(previousEntity.type()); + boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType()); + boolean intersects = textEntity.intersects(previousEntity); + if (!typeMatches || !entityTypeMatches || !intersects) { + return false; + } + } + return true; + } + + + public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) { + + if (textBlock.getTextRange().end() == textRange.end()) { + return new TextRange(textRange.end(), textRange.end()); + } + + return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java index 62da7a73..61f94301 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java @@ -5,6 +5,7 @@ import static java.util.stream.Collectors.groupingBy; import java.awt.geom.Rectangle2D; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Locale; @@ -17,6 +18,7 @@ import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.iqser.red.service.redaction.v1.server.model.ClosestEntity; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; @@ -58,28 +60,28 @@ public class EntityFindingUtility { return Optional.empty(); } - Optional optionalClosestEntity = possibleEntities.stream() + Optional optionalClosestEntity = possibleEntities.stream() .filter(entity -> pagesMatch(entity, manualEntity.getEntityPosition())) - .min(Comparator.comparingDouble(entity -> calculateMinDistance(manualEntity.getEntityPosition(), entity))); + .map(entity -> ClosestEntity.builder().distance(calculateMinDistance(manualEntity.getEntityPosition(), entity)).textEntity(entity).build()) + .min(Comparator.comparingDouble(ClosestEntity::getDistance)); if (optionalClosestEntity.isEmpty()) { log.warn("No Entity with value {} found on page {}", manualEntity.getValue(), manualEntity.getEntityPosition()); return Optional.empty(); } - TextEntity closestEntity = optionalClosestEntity.get(); - double distance = calculateMinDistance(manualEntity.getEntityPosition(), closestEntity); - if (distance > matchThreshold) { + ClosestEntity closestEntity = optionalClosestEntity.get(); + if (closestEntity.getDistance() > matchThreshold) { log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}", manualEntity.getValue(), manualEntity.getEntityPosition().get(0).pageNumber(), manualEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(), - distance, + closestEntity.getDistance(), matchThreshold); return Optional.empty(); } - return Optional.of(closestEntity); + return Optional.of(closestEntity.getTextEntity()); } @@ -97,7 +99,7 @@ public class EntityFindingUtility { } - private double calculateMinDistance(List originalPositions, TextEntity entity) { + public static double calculateMinDistance(List originalPositions, TextEntity entity) { if (originalPositions.size() != countRectangles(entity)) { return Double.MAX_VALUE; @@ -115,7 +117,7 @@ public class EntityFindingUtility { } - private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) { + private static double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) { return entity.getPositionsOnPagePerPage() .stream() @@ -128,7 +130,7 @@ public class EntityFindingUtility { } - public double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) { + public static double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) { // mirrored coordinates safe comparison double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX()); @@ -163,7 +165,10 @@ public class EntityFindingUtility { return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) .stream() - .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node)) + .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet())) + .filter(Optional::isPresent) + .map(Optional::get) + .distinct() .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT))); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java new file mode 100644 index 00000000..6b5ac081 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java @@ -0,0 +1,31 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import java.util.HashSet; +import java.util.Set; + +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; + +import lombok.Getter; + +public class IntersectingNodeVisitor implements NodeVisitor { + + @Getter + private Set intersectingNodes; + private final TextRange textRange; + + public IntersectingNodeVisitor(TextRange textRange) { + + this.textRange = textRange; + this.intersectingNodes = new HashSet<>(); + } + + @Override + public void visit(SemanticNode node) { + + if (node.getTextRange().intersects(textRange)) { + intersectingNodes.add(node); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java index 74c8ec89..6b0379f1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service.document; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -46,17 +47,24 @@ public class ManualEntityCreationService { } - public List createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, - SemanticNode node, - String dossierTemplateId) { + public List createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) { Set idRemovals = manualRedactions.getIdsToRemove(); - List manualEntities = manualRedactions.getEntriesToAdd().stream() - .filter(manualRedactionEntry -> !(idRemovals.stream().map(BaseAnnotation::getAnnotationId).toList().contains(manualRedactionEntry.getAnnotationId()) && - manualRedactionEntry.getRequestDate().isBefore(idRemovals.stream().filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())).findFirst().get().getRequestDate()))) + List manualEntities = manualRedactions.getEntriesToAdd() + .stream() + .filter(manualRedactionEntry -> !(idRemovals.stream() + .map(BaseAnnotation::getAnnotationId) + .toList() + .contains(manualRedactionEntry.getAnnotationId()) && manualRedactionEntry.getRequestDate() + .isBefore(idRemovals.stream() + .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())) + .findFirst() + .get() + .getRequestDate()))) .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary())) .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry, - dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> { + dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) + .peek(manualEntity -> { if (manualEntity.getEntityType().equals(EntityType.HINT)) { manualEntity.skip("MAN.5.1", "manual hint is skipped by default"); } else { @@ -75,12 +83,12 @@ public class ManualEntityCreationService { List notFoundManualEntities = new LinkedList<>(); for (ManualEntity manualEntity : manualEntities) { - Optional optionalRedactionEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD); - if (optionalRedactionEntity.isEmpty()) { + Optional optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD); + if (optionalClosestEntity.isEmpty()) { notFoundManualEntities.add(manualEntity); continue; } - createCorrectEntity(manualEntity, node, optionalRedactionEntity.get().getTextRange()); + createCorrectEntity(manualEntity, optionalClosestEntity.get()); } tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); return notFoundManualEntities; @@ -90,25 +98,29 @@ public class ManualEntityCreationService { /** * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters. * - * @param manualEntity The entity identifier for the RedactionEntity. - * @param node The SemanticNode associated with the RedactionEntity. - * @param closestTextRange The closest Boundary to the RedactionEntity. + * @param manualEntity The entity identifier for the RedactionEntity. + * @param closestEntity The closest Boundary to the RedactionEntity. */ - private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) { + private void createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) { - TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node); + TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId()); + + correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); + correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); + correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges())); + correctEntity.setPages(new HashSet<>(closestEntity.getPages())); + + correctEntity.setValue(closestEntity.getValue()); + correctEntity.setTextAfter(closestEntity.getTextAfter()); + correctEntity.setTextBefore(closestEntity.getTextBefore()); + + correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); + correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); - - // AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly! - List redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size()); - for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) { - redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine())); - } - correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java index 985d58b1..efdc7d24 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java @@ -10,8 +10,8 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import io.micrometer.observation.annotation.Observed; import lombok.RequiredArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java new file mode 100644 index 00000000..a027c47a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java @@ -0,0 +1,8 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; + +public interface NodeVisitor { + + void visit(SemanticNode node); +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index 17f809f0..88268f3a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -42,7 +42,7 @@ public class EntityDroolsExecutionService { EntityEnrichmentService entityEnrichmentService; ObservationRegistry observationRegistry; - + ManualChangesApplicationService manualChangesApplicationService; RedactionServiceSettings settings; @@ -73,7 +73,6 @@ public class EntityDroolsExecutionService { KieSession kieSession = kieContainer.newKieSession(); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); - ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService); kieSession.setGlobal("document", document); kieSession.setGlobal("entityCreationService", entityCreationService); @@ -88,7 +87,7 @@ public class EntityDroolsExecutionService { fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert); if (manualRedactions != null) { - manualRedactions.getResizeRedactions().stream().filter(manualResizeRedaction -> !manualResizeRedaction.getUpdateDictionary()).forEach(kieSession::insert); + manualRedactions.getResizeRedactions().forEach(kieSession::insert); manualRedactions.getRecategorizations().forEach(kieSession::insert); manualRedactions.getEntriesToAdd().forEach(kieSession::insert); manualRedactions.getForceRedactions().forEach(kieSession::insert); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index a7f3237b..e10a911b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -293,7 +293,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { @Test public void titleExtraction() throws IOException { - AnalyzeRequest request = uploadFileToStorage("files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); System.out.println("Finished structure analysis"); @@ -1331,6 +1331,36 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED); } + @Test + @SneakyThrows + public void testResizeWithUpdateDictionaryTrue() { + + String pdfFile = "files/new/crafted document.pdf"; + + AnalyzeRequest request = uploadFileToStorage(pdfFile); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get(); + + request.setManualRedactions(ManualRedactions.builder() + .resizeRedactions(Set.of(ManualResizeRedaction.builder() + .updateDictionary(true) + .annotationId(david.getId()) + .requestDate(OffsetDateTime.now()) + .value("David Ksenia") + .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build())) + .addToAllDossiers(false) + .build())) + .build()); + analyzeService.reanalyze(request); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var resizedEntity = entityLog.getEntityLogEntry().stream().filter(e -> e.getId().equals(david.getId())).findFirst().get(); + assertEquals(resizedEntity.getState(), EntryState.APPLIED); + assertEquals(resizedEntity.getValue(), "David Ksenia"); + } + private IdRemoval getIdRemoval(String id) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java index c6e30b9a..95174da1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java @@ -5,16 +5,18 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import org.junit.jupiter.api.Test; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; public class TextEntityTest { @Test public void testMatchedRule() { - TextEntity entity = TextEntity.initialEntityNode(new TextRange(1, 100), "PII", EntityType.ENTITY); + ManualEntity entity = ManualEntity.builder() + .type("PII") + .entityType(EntityType.ENTITY) + .build(); entity.skip("CBI.1.0", ""); entity.skip("CBI.2.0", ""); entity.skip("CBI.3.0", ""); @@ -29,7 +31,10 @@ public class TextEntityTest { @Test public void testMatchedRuleWithNonsense() { - TextEntity entity = TextEntity.initialEntityNode(new TextRange(1, 100), "PII", EntityType.ENTITY); + ManualEntity entity = ManualEntity.builder() + .type("PII") + .entityType(EntityType.ENTITY) + .build(); assertThrows(IllegalArgumentException.class, () -> { entity.skip("", ""); }); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java index 38a52644..739eb54c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java @@ -49,6 +49,21 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); } + @Test + public void assertEntitiesAreDuplicatedWithTheirTableCell() { + + Document document = buildGraph("files/Minimal Examples/Meto1_Page22.pdf"); + List entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document).toList(); + assertEquals(3, entities.size()); + assertEquals(1, entities.stream().distinct().count()); + assertEquals(2, entities.get(0).getDuplicateTextRanges().size()); + + var node = entities.get(0).getDeepestFullyContainingNode(); + + assertTrue(node.getTextRange().contains(entities.get(0).getTextRange())); + assertTrue(entities.get(0).getDuplicateTextRanges().stream().allMatch(tr -> node.getTextRange().contains(tr))); + } + @Test public void assertCollectAllEntitiesWorks() { @@ -82,7 +97,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr assert start != -1; TextRange textRange = new TextRange(start, start + searchTerm.length()); - TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY); + TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document); entityCreationService.addEntityToGraph(textEntity, document); return textEntity; } @@ -241,7 +256,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr assert start != -1; TextRange textRange = new TextRange(start, start + searchTerm.length()); - TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY); + TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document); entityCreationService.addEntityToGraph(textEntity, document); assertEquals("2.6.1 Summary of ", textEntity.getTextBefore()); @@ -299,7 +314,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr assert start != -1; TextRange textRange = new TextRange(start, start + searchTerm.length()); - TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY); + TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document); entityCreationService.addEntityToGraph(textEntity, document); Page pageNode = document.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 0bd06cfb..e3e6a69c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -1,7 +1,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph; import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; -import static org.mockito.Mockito.doThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import java.awt.Color; @@ -34,6 +34,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; @@ -41,16 +45,11 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider; -import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; -import com.iqser.red.service.redaction.v1.server.model.NerEntities; -import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; +import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.SneakyThrows; @@ -245,8 +244,12 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration System.out.printf("%d Searches took %s s, average %.2f ms\n", numberOfRuns, ((float) totalSearchTime / 1000), totalSearchTime / numberOfRuns); System.out.printf("%d Insertions took %s s, average %.2f ms\n", numberOfRuns, ((float) totalInsertTime / 1000), totalInsertTime / numberOfRuns); System.out.printf("Found %d entities and saved %d\n", foundEntities.size(), document.getEntities().size()); - assert document.getEntities().size() == foundEntities.size(); - + for (TextEntity entity : document.getEntities()) { + var foundEntity = foundEntities.stream().filter(f -> f.getId().equals(entity.getId())).findFirst().get(); + assertTrue(foundEntity.getTextRange().equals(entity.getTextRange()) || foundEntity.getDuplicateTextRanges().contains(entity.getTextRange())); + } + assert document.getEntities().stream().mapToInt(e -> e.getDuplicateTextRanges().size() + 1).sum() == foundEntities.size(); + assert foundEntities.stream().map(TextEntity::getId).distinct().count() == document.getEntities().size(); drawAllEntities(filename, document); } @@ -300,7 +303,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration searchImplementation.getBoundaries(textBlock, textBlock.getTextRange()) .stream() .filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary)) - .map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType)) + .map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType, document)) .forEach(foundEntities::add); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index f642b81a..128849f7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -4,11 +4,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.when; +import static org.wildfly.common.Assert.assertTrue; import java.awt.geom.Rectangle2D; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.OffsetDateTime; import java.util.Comparator; @@ -121,6 +123,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { loadNerForTest(); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse()); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() @@ -147,7 +150,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { String filePath = "files/new/crafted document.pdf"; AnalyzeRequest request = uploadFileToStorage(filePath); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); - AnalyzeResult result = analyzeService.analyze(request); + analyzeService.analyze(request); String testEntityValue1 = "Desiree"; String testEntityValue2 = "Melanie"; @@ -157,7 +160,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)); String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."; - TextEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get(); + entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get(); String idToResize = redactionLog.getEntityLogEntry() .stream() @@ -165,24 +168,18 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .max(Comparator.comparingInt(EntityLogEntry::getStartOffset)) .get() .getId(); - List resizedPositions = expandedEntity.getPositionsOnPagePerPage() - .get(0) - .getRectanglePerLine() - .stream() - .map(rectangle2D -> toAnnotationRectangle(rectangle2D, 3)) - .toList(); - ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() - .annotationId(idToResize) - .value(expandedEntityKeyword) - .positions(resizedPositions) - .requestDate(OffsetDateTime.now()) - .updateDictionary(false) - .build(); - manualResizeRedaction.setUpdateDictionary(false); ManualRedactions manualRedactions = new ManualRedactions(); - manualRedactions.getResizeRedactions().add(manualResizeRedaction); + manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder() + .annotationId(idToResize) + .value(expandedEntityKeyword) + .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(), + Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build())) + .addToAllDossiers(false) + .updateDictionary(false) + .requestDate(OffsetDateTime.now()) + .build()); request.setManualRedactions(manualRedactions); - AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request); + analyzeService.reanalyze(request); redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf"); @@ -225,7 +222,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") .legalBasis("Something") - .build())); + .build())); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); manualRedactionEntry.setAnnotationId(manualAddId); @@ -248,7 +245,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { .annotationId("675eba69b0c2917de55462c817adaa05") .fileId("fileId") .legalBasis("Manual Legal Basis Change") - .requestDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) .build()))); analyzeService.reanalyze(request); @@ -267,6 +264,45 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { } + @Test + public void testManualRedactionIsDuplicatedWithTableCells() throws IOException { + + System.out.println("testManualRedaction"); + long start = System.currentTimeMillis(); + String pdfFile = "files/Minimal Examples/Meto1_Page22.pdf"; + + ManualRedactions manualRedactions = new ManualRedactions(); + + String manualAddId = UUID.randomUUID().toString(); + + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId(manualAddId); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setType("name"); + manualRedactionEntry.setValue("Surface Water"); + manualRedactionEntry.setReason("Dictionary Request"); + manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(76.584f).topLeftY(506.56238f).width(57.4329f).height(9.065156f).page(1).build())); + manualRedactions.setEntriesToAdd(Set.of(manualRedactionEntry)); + AnalyzeRequest request = uploadFileToStorage(pdfFile); + request.setManualRedactions(manualRedactions); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + AnalyzeResult result = analyzeService.analyze(request); + + var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); + + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/" + Path.of(pdfFile).getFileName() + "MANUAL_REDACTION_TEST.pdf")) { + fileOutputStream.write(annotateResponse.getDocument()); + } + long end = System.currentTimeMillis(); + var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny(); + assertTrue(optionalEntry.isPresent()); + assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell + System.out.println("duration: " + (end - start)); + System.out.println("numberOfPages: " + result.getNumberOfPages()); + } + + @Test public void testReCategorizeToVertebrateChangesCbiAuthor() { @@ -296,7 +332,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { ManualRecategorization recategorization = ManualRecategorization.builder() .requestDate(OffsetDateTime.now()) - .type("vertebrate") + .type("vertebrate") .annotationId(oxfordUniversityPress.getId()) .fileId(TEST_FILE_ID) .build(); @@ -345,18 +381,19 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder() .annotationId(annotationId) .requestDate(OffsetDateTime.now()) + .type("manual") .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated") .positions(List.of(// new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), // new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), // new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) // - .build())); + .build())); ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder() .annotationId(annotationId) .requestDate(OffsetDateTime.now()) .value("Expand to Hint") .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2))) - .updateDictionary(false) + .updateDictionary(false) .build(); manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction)); request.setManualRedactions(manualRedactions); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index 4d59e8d0..860f4d1b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -7,12 +7,14 @@ import static org.wildfly.common.Assert.assertFalse; import java.awt.geom.Rectangle2D; import java.time.OffsetDateTime; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; +import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; @@ -50,7 +52,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { assertEquals(biggerEntity.getTextRange(), entity.getTextRange()); assertEquals(biggerEntity.getDeepestFullyContainingNode(), entity.getDeepestFullyContainingNode()); - assertEquals(biggerEntity.getIntersectingNodes(), entity.getIntersectingNodes()); + assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty()); assertEquals(biggerEntity.getPages(), entity.getPages()); assertEquals(biggerEntity.getValue(), entity.getValue()); assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index c681b6fb..85d8311c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -119,7 +119,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { entity.getManualOverwrite().addChange(imageRecategorizationRequest); assertTrue(entity.getManualOverwrite().getRecategorized().isPresent()); assertTrue(entity.getManualOverwrite().getRecategorized().get()); - assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.getType())); + assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index c5ae5d41..c507b417 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -167,14 +167,14 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { private List getPositionsFromEntityOfType(String type, List entities) { - return getPositionsFromEntities(entities.stream().filter(e -> e.getType().equals(type))); + return getPositionsFromEntities(entities.stream().filter(e -> e.type().equals(type))); } private List getPositionsFromEntityNotOfType(List types, List entities) { - return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.getType().equals(type)))); + return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.type().equals(type)))); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java index 8ce45581..b6cbf355 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java @@ -28,6 +28,8 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { @Autowired protected EntityEnrichmentService entityEnrichmentService; + @Autowired + protected ManualChangesApplicationService manualChangesApplicationService; protected EntityCreationService entityCreationService; protected KieSession kieSession; @@ -72,7 +74,6 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { kieSession = kieContainer.newKieSession(); entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); - ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService); kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); kieSession.setGlobal("entityCreationService", entityCreationService); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java index b787d489..8eeba13c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java @@ -174,7 +174,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT analyzeService.reanalyze(request); - List positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).build()); + List positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).page(1).build()); ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "was above the AOEL"); request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); @@ -221,7 +221,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeService.analyze(request); - List positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build()); + List positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build()); List positions2 = List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(80.144233125f).height(10.048125f).page(1).build()); List positions3 = List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(291.5095f).width(107.01071999999994f).height(10.048125f).page(1).build()); ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL"); @@ -285,7 +285,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeService.analyze(request); - List positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build()); + List positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build()); ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL"); request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); @@ -323,7 +323,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeService.analyze(request); - List positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).build()); + List positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).page(1).build()); ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AAOEL"); request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); @@ -361,7 +361,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeService.analyze(request); - List positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build()); + List positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build()); ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "Does Not Exist"); request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 857f3060..7c19f205 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -84,7 +84,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -92,7 +92,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -102,7 +102,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_address", dictionaryEntry) + $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); end @@ -110,7 +110,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" rule "CBI.1.1: Redact CBI Address (vertebrate Study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_address", dictionaryEntry) + $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -119,7 +119,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)" // Rule unit: CBI.2 rule "CBI.2.0: Do not redact genitive CBI Author" when - $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s")) + $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s")) then entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); @@ -148,7 +148,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList() $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities() + $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() then $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress)); end @@ -301,7 +301,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC rule "PII.0.0: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -309,7 +309,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -625,7 +625,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.12.0: Expand PII entities with salutation prefix" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); @@ -634,7 +634,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" rule "PII.12.1: Expand PII entities with salutation prefix" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); @@ -693,7 +693,7 @@ rule "ETC.3.1: Redact logos (vertebrate study)" rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'" when not FileAttribute(label == "Confidentiality", value == "confidential") - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); @@ -709,7 +709,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -733,7 +733,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -811,13 +811,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -825,7 +824,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -883,8 +882,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -895,8 +894,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.1.0: Merge intersecting Entities of same type" salience 64 when - $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) + $first: TextEntity($type: type(), $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -911,8 +910,8 @@ rule "X.1.0: Merge intersecting Entities of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -924,8 +923,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -936,8 +935,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -959,8 +958,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); @@ -971,8 +970,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); @@ -982,8 +981,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 256a327a..156f3f09 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -97,7 +97,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -105,7 +105,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -115,7 +115,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_address", dictionaryEntry) + $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); end @@ -123,7 +123,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" rule "CBI.1.1: Redact CBI Address (vertebrate Study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_address", dictionaryEntry) + $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -132,7 +132,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)" // Rule unit: CBI.2 rule "CBI.2.0: Do not redact genitive CBI Author" when - $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s")) + $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s")) then entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); @@ -331,7 +331,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList() $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities() + $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() then $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress)); end @@ -493,7 +493,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V rule "CBI.13.0: Ignore CBI Address recommendations" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION) + $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION) then $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); retract($entity) @@ -503,7 +503,7 @@ rule "CBI.13.0: Ignore CBI Address recommendations" // Rule unit: CBI.14 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" when - $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at")) + $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at")) then $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)"); end @@ -606,7 +606,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with rule "CBI.18.0: Expand CBI_author entities with firstname initials" no-loop true when - $entityToExpand: TextEntity(type == "CBI_author", + $entityToExpand: TextEntity(type() == "CBI_author", value.matches("[^\\s]+"), textAfter.startsWith(" "), anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") @@ -624,7 +624,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" // Rule unit: CBI.19 rule "CBI.19.0: Expand CBI_author entities with salutation prefix" when - $entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> { @@ -668,7 +668,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule - not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities() + not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities() then entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { @@ -680,7 +680,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule - not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities() + not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities() then entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { @@ -707,7 +707,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n rule "PII.0.0: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -715,7 +715,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -1047,7 +1047,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.12.0: Expand PII entities with salutation prefix" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); @@ -1134,21 +1134,21 @@ rule "ETC.3.1: Redact logos (vertebrate study)" // Rule unit: ETC.4 rule "ETC.4.0: Redact dossier dictionary entries" when - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end rule "ETC.4.1: Redact dossier dictionary entries" when - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); end rule "ETC.4.2: Redact dossier dictionary entries" when - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); end @@ -1158,7 +1158,7 @@ rule "ETC.4.2: Redact dossier dictionary entries" rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'" when not FileAttribute(label == "Confidentiality", value == "confidential") - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); @@ -1212,7 +1212,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" rule "ETC.9.0: Redact skipped impurities" when FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y") - $skippedImpurities: TextEntity(type == "skipped_impurities") + $skippedImpurities: TextEntity(type() == "skipped_impurities") then $skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); end @@ -1220,7 +1220,7 @@ rule "ETC.9.0: Redact skipped impurities" rule "ETC.9.1: Redact impurities" when FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y") - $skippedImpurities: TextEntity(type == "impurities") + $skippedImpurities: TextEntity(type() == "impurities") then $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); end @@ -1229,7 +1229,7 @@ rule "ETC.9.1: Redact impurities" // Rule unit: ETC.10 rule "ETC.10.0: Redact Product Composition Information" when - $compositionInformation: TextEntity(type == "product_composition") + $compositionInformation: TextEntity(type() == "product_composition") then $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009"); end @@ -1256,7 +1256,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -1278,7 +1278,7 @@ rule "AI.2.0: Add all NER Entities of any type except CBI_author" then nerEntities.getNerEntityList().stream() .filter(nerEntity -> !nerEntity.type().equals("CBI_author")) - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document)); end @@ -1289,7 +1289,7 @@ rule "AI.3.0: Recommend authors from AI as PII" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document)); end @@ -1303,7 +1303,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -1381,13 +1381,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -1395,7 +1394,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -1453,8 +1452,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -1465,8 +1464,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.1.0: Merge intersecting Entities of same type" salience 64 when - $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) + $first: TextEntity($type: type(), $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -1481,8 +1480,8 @@ rule "X.1.0: Merge intersecting Entities of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1494,8 +1493,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -1506,8 +1505,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -1529,8 +1528,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); @@ -1541,8 +1540,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); @@ -1552,8 +1551,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 72c1cf67..9cf983bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -466,7 +466,9 @@ rule "DOC.7.1: Performing Laboratory (Country)" then nerEntities.streamEntitiesOfType("COUNTRY") .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange())) - .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section)) + .map(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section)) + .flatMap(Optional::stream) + .collect(Collectors.toList()) .forEach(entity -> { entity.apply("DOC.7.1", "Performing Laboratory found", "n-a"); }); @@ -475,7 +477,7 @@ rule "DOC.7.1: Performing Laboratory (Country)" rule "DOC.7.2: Performing Laboratory (Country & Name) from dict" when $section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:"))) - $countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities() + $countryOrNameFromDictionary: TextEntity(type() == "laboratory_country" || type() == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities() then $countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found."); end @@ -1161,7 +1163,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -1239,13 +1241,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -1253,7 +1254,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -1311,8 +1312,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -1323,8 +1324,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1336,8 +1337,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -1348,8 +1349,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -1371,8 +1372,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index c0750bcc..50272f51 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -75,7 +75,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -153,13 +153,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -167,7 +166,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -225,8 +224,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index bba749d1..3e5a6bb5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -339,7 +339,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V // Rule unit: CBI.14 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" when - $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at")) + $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at")) then $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)"); end @@ -442,7 +442,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with rule "CBI.18.0: Expand CBI_author entities with firstname initials" no-loop true when - $entityToExpand: TextEntity(type == "CBI_author", + $entityToExpand: TextEntity(type() == "CBI_author", value.matches("[^\\s]+"), textAfter.startsWith(" "), anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") @@ -460,7 +460,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" // Rule unit: CBI.19 rule "CBI.19.0: Expand CBI_author entities with salutation prefix" when - $entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> { @@ -505,7 +505,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC rule "PII.0.0: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -513,7 +513,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -733,7 +733,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.12.0: Expand PII entities with salutation prefix" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); @@ -742,7 +742,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" rule "PII.12.1: Expand PII entities with salutation prefix" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); @@ -800,7 +800,7 @@ rule "ETC.3.1: Redact logos (vertebrate study)" // Rule unit: ETC.4 rule "ETC.4.0: Redact dossier dictionary entries" when - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -810,7 +810,7 @@ rule "ETC.4.0: Redact dossier dictionary entries" rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'" when not FileAttribute(label == "Confidentiality", value == "confidential") - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); @@ -869,7 +869,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -893,7 +893,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -971,13 +971,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -985,7 +984,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -1043,8 +1042,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -1055,8 +1054,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.1.0: Merge intersecting Entities of same type" salience 64 when - $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) + $first: TextEntity($type: type(), $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -1071,8 +1070,8 @@ rule "X.1.0: Merge intersecting Entities of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1084,8 +1083,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -1096,8 +1095,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -1119,8 +1118,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); @@ -1131,8 +1130,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); @@ -1142,8 +1141,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 61d808dd..e7e940d5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -71,7 +71,7 @@ query "getFileAttributes" rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -83,7 +83,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" rule "PII.0.0: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -98,7 +98,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -112,7 +112,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -190,13 +190,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -204,7 +203,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -249,8 +248,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -261,8 +260,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.1.0: Merge intersecting Entities of same type" salience 64 when - $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) + $first: TextEntity($type: type(), $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -277,8 +276,8 @@ rule "X.1.0: Merge intersecting Entities of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -290,8 +289,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -302,8 +301,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -325,8 +324,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); @@ -337,8 +336,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); @@ -348,8 +347,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 888be2a8..73c345d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -225,7 +225,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -303,13 +303,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -317,7 +316,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -375,8 +374,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -387,8 +386,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -400,8 +399,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -412,8 +411,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -435,8 +434,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); @@ -447,8 +446,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); @@ -458,8 +457,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 62075dab..ab04310c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -125,7 +125,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -203,13 +203,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -217,7 +216,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -275,8 +274,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -287,8 +286,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.1.0: Merge intersecting Entities of same type" salience 64 when - $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) + $first: TextEntity($type: type(), $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -303,8 +302,8 @@ rule "X.1.0: Merge intersecting Entities of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -316,8 +315,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -328,8 +327,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -351,8 +350,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); @@ -363,8 +362,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); @@ -374,8 +373,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Meto1_Page22.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Meto1_Page22.pdf new file mode 100644 index 00000000..e45c3042 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Meto1_Page22.pdf differ diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index 1d96d377..8b28390c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -81,7 +81,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -89,7 +89,7 @@ rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)" rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)" when FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -99,7 +99,7 @@ rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)" rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $entity: TextEntity(type == "CBI_address", dictionaryEntry) + $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); end @@ -107,7 +107,7 @@ rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)" rule "CBI.1.1: Redact CBI Address (Vertebrate Study)" when FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $entity: TextEntity(type == "CBI_address", dictionaryEntry) + $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then $entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -116,7 +116,7 @@ rule "CBI.1.1: Redact CBI Address (Vertebrate Study)" // Rule unit: CBI.2 rule "CBI.2.0: Don't redact genitive CBI_author" when - $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied()) + $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied()) then entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); @@ -145,7 +145,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author")) $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList() $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList() - $author: TextEntity(type == "CBI_author", active()) from $tableCell.getEntities() + $author: TextEntity(type() == "CBI_author", active()) from $tableCell.getEntities() then $author.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $author)); end @@ -298,7 +298,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC rule "PII.0.0: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -306,7 +306,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -452,7 +452,7 @@ rule "ETC.3.1: Redact logos (non vertebrate study)" rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'" when not FileAttribute(label == "Confidentiality", value == "confidential") - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); update($dossierRedaction); @@ -469,7 +469,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -494,7 +494,7 @@ rule "MAN.0.0: Apply manual resize redaction" $resizeRedaction: ManualResizeRedaction($id: annotationId) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -575,8 +575,7 @@ rule "MAN.3.0: Apply entity recategorization" $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); + update($entityToBeRecategorized); end rule "MAN.3.1: Apply image recategorization" @@ -617,8 +616,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !resized(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -629,8 +628,8 @@ rule "X.0.0: remove Entity contained by Entity of same type" rule "X.1.0: merge intersecting Entities of same type" salience 64 when - $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active()) + $first: TextEntity($type: type(), $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !resized(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -645,8 +644,8 @@ rule "X.1.0: merge intersecting Entities of same type" rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, entityType == EntityType.ENTITY, !resized(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -658,8 +657,8 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -670,8 +669,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.ENTITY, active()) + $recommendation: TextEntity(intersects($entity), type() == $type, entityType == EntityType.RECOMMENDATION, !resized(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); @@ -695,8 +694,8 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active()) + $higherRank: TextEntity($type: type(), entityType == EntityType.ENTITY, active()) + $lowerRank: TextEntity(intersects($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY"); diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index c885ae6f..7214d931 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -97,7 +97,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -105,7 +105,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_author", dictionaryEntry) + $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -115,7 +115,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_address", dictionaryEntry) + $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); end @@ -123,7 +123,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" rule "CBI.1.1: Redact CBI Address (vertebrate Study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_address", dictionaryEntry) + $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -132,7 +132,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)" // Rule unit: CBI.2 rule "CBI.2.0: Do not redact genitive CBI Author" when - $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s")) + $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s")) then entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document) .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found")); @@ -331,7 +331,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList() $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList() - $authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities() + $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities() then $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress)); end @@ -493,7 +493,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V rule "CBI.13.0: Ignore CBI Address recommendations" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION) + $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION) then $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); retract($entity) @@ -503,7 +503,7 @@ rule "CBI.13.0: Ignore CBI Address recommendations" // Rule unit: CBI.14 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" when - $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at")) + $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at")) then $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)"); end @@ -606,7 +606,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with rule "CBI.18.0: Expand CBI_author entities with firstname initials" no-loop true when - $entityToExpand: TextEntity(type == "CBI_author", + $entityToExpand: TextEntity(type() == "CBI_author", value.matches("[^\\s]+"), textAfter.startsWith(" "), anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)") @@ -624,7 +624,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" // Rule unit: CBI.19 rule "CBI.19.0: Expand CBI_author entities with salutation prefix" when - $entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> { @@ -667,7 +667,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule - not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities() + not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities() then entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { @@ -679,7 +679,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule - not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities() + not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities() then entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { @@ -706,7 +706,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n rule "PII.0.0: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -714,7 +714,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $pii: TextEntity(type == "PII", dictionaryEntry) + $pii: TextEntity(type() == "PII", dictionaryEntry) then $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -1045,7 +1045,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.12.0: Expand PII entities with salutation prefix" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); @@ -1056,7 +1056,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" rule "PII.12.1: Expand PII entities with salutation prefix" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) + $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); @@ -1142,21 +1142,21 @@ rule "ETC.3.1: Redact logos (vertebrate study)" // Rule unit: ETC.4 rule "ETC.4.0: Redact dossier dictionary entries" when - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end rule "ETC.4.1: Redact dossier dictionary entries" when - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); end rule "ETC.4.2: Redact dossier dictionary entries" when - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); end @@ -1166,7 +1166,7 @@ rule "ETC.4.2: Redact dossier dictionary entries" rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'" when not FileAttribute(label == "Confidentiality", value == "confidential") - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential"); $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); @@ -1219,7 +1219,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" rule "ETC.9.0: Redact skipped impurities" when FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y") - $skippedImpurities: TextEntity(type == "skipped_impurities") + $skippedImpurities: TextEntity(type() == "skipped_impurities") then $skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); end @@ -1227,7 +1227,7 @@ rule "ETC.9.0: Redact skipped impurities" rule "ETC.9.1: Redact impurities" when FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y") - $skippedImpurities: TextEntity(type == "impurities") + $skippedImpurities: TextEntity(type() == "impurities") then $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); end @@ -1235,7 +1235,7 @@ rule "ETC.9.1: Redact impurities" // Rule unit: ETC.10 rule "ETC.10.0: Redact Product Composition Information" when - $compositionInformation: TextEntity(type == "product_composition") + $compositionInformation: TextEntity(type() == "product_composition") then $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009"); end @@ -1256,7 +1256,7 @@ rule "ETC.11.0: Recommend first line in table cell with name and address of owne rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.12.0", "Dossier dictionary entry found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); end @@ -1264,7 +1264,7 @@ rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)" rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)" when FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") - $dossierRedaction: TextEntity(type == "dossier_redaction") + $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); end @@ -1278,7 +1278,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document)); end @@ -1300,7 +1300,7 @@ rule "AI.2.0: Add all NER Entities of any type except CBI_author" then nerEntities.getNerEntityList().stream() .filter(nerEntity -> !nerEntity.type().equals("CBI_author")) - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document)); end @@ -1311,7 +1311,7 @@ rule "AI.3.0: Recommend authors from AI as PII" nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) then nerEntities.streamEntitiesOfType("CBI_author") - .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document)); + .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document)); end //------------------------------------ Manual redaction rules ------------------------------------ @@ -1324,7 +1324,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -1402,13 +1402,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -1416,7 +1415,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -1474,8 +1473,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -1486,8 +1485,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.1.0: Merge intersecting Entities of same type" salience 64 when - $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) - $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) + $first: TextEntity($type: type(), $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) then TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); $first.remove("X.1.0", "merge intersecting Entities of same type"); @@ -1502,8 +1501,8 @@ rule "X.1.0: Merge intersecting Entities of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1515,8 +1514,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -1527,8 +1526,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -1552,8 +1551,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); @@ -1564,8 +1563,8 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" salience 32 when - $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) then $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); @@ -1576,8 +1575,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" salience 32 when - $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) - $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) + $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges()) + $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length()) then $higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index 33dd8253..a8ac8685 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -464,7 +464,9 @@ rule "DOC.7.1: Performing Laboratory (Country)" then nerEntities.streamEntitiesOfType("COUNTRY") .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange())) - .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section)) + .map(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section)) + .flatMap(Optional::stream) + .collect(Collectors.toList()) .forEach(entity -> { entity.apply("DOC.7.1", "Performing Laboratory found", "n-a"); }); @@ -473,7 +475,7 @@ rule "DOC.7.1: Performing Laboratory (Country)" rule "DOC.7.2: Performing Laboratory (Country & Name) from dict" when $section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:"))) - $countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities() + $countryOrNameFromDictionary: TextEntity(type() == "laboratory_country" || type() == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities() then $countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found."); end @@ -1308,7 +1310,7 @@ rule "MAN.0.0: Apply manual resize redaction" not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) $entityToBeResized: TextEntity(matchesAnnotationId($id)) then - manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); update($entityToBeResized); $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); @@ -1386,13 +1388,12 @@ rule "MAN.3.0: Apply entity recategorization" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + update($entityToBeRecategorized); retract($recategorization); - // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. - retract($entityToBeRecategorized); end rule "MAN.3.1: Apply entity recategorization of same type" @@ -1400,7 +1401,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" when $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate) not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) - $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); retract($recategorization); @@ -1457,8 +1458,8 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type, $entityType: entityType, active()) - $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + $larger: TextEntity($type: type(), $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); retract($contained); @@ -1469,8 +1470,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) then $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); @@ -1482,8 +1483,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); retract($recommendation); @@ -1494,8 +1495,8 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $entity.addEngines($recommendation.getEngines()); $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); @@ -1519,8 +1520,8 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type, entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) then $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); retract($recommendation); diff --git a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt index 92fa5519..da6022cd 100644 --- a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt +++ b/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt @@ -1 +1 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;\nimport com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualRedactionApplicationService manualRedactionApplicationService\nglobal NerEntitiesAdapter nerEntitiesAdapter\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//------------------------------------ Syngenta specific rules ------------------------------------\n\n// Rule unit: SYN.1\nrule \"SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n $section: Section(containsString(\"CT\") || containsString(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n entityCreationService.byRegexIgnoreCase(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.skip(\"SYN.1.0\", \"\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n\n\n//------------------------------------ CBI rules ------------------------------------\n\n// Rule unit: CBI.0\nrule \"CBI.0.0: Redact CBI Authors (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.0\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"CBI.0.1: Redact CBI Authors (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.1\nrule \"CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.skip(\"CBI.1.0\", \"Address found for Non Vertebrate Study\");\n end\n\nrule \"CBI.1.1: Redact CBI Address (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.apply(\"CBI.1.1\", \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.2\nrule \"CBI.2.0: Don't redact genitive CBI_author\"\n when\n $entity: RedactionEntity(type == \"CBI_author\", anyMatch(textAfter, \"['’’'ʼˈ´`‘′ʻ’']s\"), isApplied())\n then\n RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), \"CBI_author\", EntityType.FALSE_POSITIVE, document);\n falsePositive.skip(\"CBI.2.0\", \"Genitive Author found\");\n insert(falsePositive);\n end\n\n\n// Rule unit: CBI.7\nrule \"CBI.7.0: Do not redact Names and Addresses if published information found in section without tables\"\n when\n $section: Section(!hasTables(),\n hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $section.getEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.0\",\n \"Published Information found in section\",\n $section.getEntitiesOfType(\"published_information\")\n );\n });\n end\n\nrule \"CBI.7.1: Do not redact Names and Addresses if published information found in same table row\"\n when\n $table: Table(hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.1\",\n \"Published Information found in row\",\n $table.getEntitiesOfTypeInSameRow(\"published_information\", redactionEntity)\n );\n });\n end\n\n\n// Rule unit: CBI.9\nrule \"CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.0\", \"Author(s) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.1\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.10\nrule \"CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.0\", \"Author(s) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.11\nrule \"CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -1\n when\n $table: Table(hasHeader(\"Author(s)\") && hasHeader(\"Vertebrate Study Y/N\"))\n then\n $table.getEntitiesOfType(\"CBI_author\").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));\n end\n\n\n// Rule unit: CBI.16\nrule \"CBI.16.0: Add CBI_author with \\\"et al.\\\" Regex (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.0\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n insert(entity);\n });\n end\n\nrule \"CBI.16.1: Add CBI_author with \\\"et al.\\\" Regex (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.1\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n });\n end\n\n\n// Rule unit: CBI.17\nrule \"CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species\") && containsString(\"Source\") && !containsString(\"Species:\") && !containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.0\", \"Line after \\\"Source\\\" in Test Organism Section\");\n insert(entity);\n });\n end\n\nrule \"CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species:\"), containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source:\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.1\", \"Line after \\\"Source:\\\" in Test Animals Section\");\n insert(entity);\n });\n end\n\n\n// Rule unit: CBI.20\nrule \"CBI.20.0: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.skip(\"CBI.20.0\", \"PERFORMING LABORATORY was found for non vertebrate study\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\nrule \"CBI.20.1: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.apply(\"CBI.20.1\", \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\n\n//------------------------------------ PII rules ------------------------------------\n\n// Rule unit: PII.0\nrule \"PII.0.0: Redact all PII (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.0\", \"Personal Information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"PII.0.1: Redact all PII (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.1\", \"Personal Information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: PII.1\nrule \"PII.1.0: Redact Emails by RegEx (Non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.0\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\nrule \"PII.1.1: Redact Emails by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.1\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\n\n// Rule unit: PII.2\nrule \"PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.0\", \"Found by Phone and Fax Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\nrule \"PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.1\", \"Found by Phone and Fax Regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\n\n// Rule unit: PII.9\nrule \"PII.9.0: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.0\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.1: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.1\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.2: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.2\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.3: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.3\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\n\n//------------------------------------ Other rules ------------------------------------\n\n// Rule unit: ETC.0\nrule \"ETC.0.0: Purity Hint\"\n when\n $section: Section(containsStringIgnoreCase(\"purity\"))\n then\n entityCreationService.byRegexIgnoreCase(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", \"hint_only\", EntityType.ENTITY, 1, $section)\n .forEach(hint -> {\n hint.addEngine(Engine.RULE);\n hint.skip(\"ETC.0.0\", \"\");\n });\n end\n\n\n// Rule unit: ETC.2\nrule \"ETC.2.0: Redact signatures (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.2.0: Redact signatures (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.3\nrule \"ETC.3.0: Redact logos (vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.0\", \"Logo Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.3.1: Redact logos (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.1\", \"Logo Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.5\nrule \"ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n not FileAttribute(label == \"Confidentiality\", value == \"confidential\")\n $dossierRedaction: RedactionEntity(type == \"dossier_redaction\")\n then\n $dossierRedaction.removeFromGraph();\n retract($dossierRedaction);\n end\n\n\n//------------------------------------ AI rules ------------------------------------\n\n// Rule unit: AI.0\nrule \"AI.0.0: add all NER Entities of type CBI_author\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"CBI_author\"))\n then\n nerEntities.streamEntitiesOfType(\"CBI_author\")\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))\n .forEach(entity -> insert(entity));\n end\n\n\n// Rule unit: AI.1\nrule \"AI.1.0: combine and add NER Entities as CBI_address\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"ORG\") || hasEntitiesOfType(\"STREET\") || hasEntitiesOfType(\"CITY\"))\n then\n nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)\n .map(boundary -> entityCreationService.byBoundary(boundary, \"CBI_address\", EntityType.RECOMMENDATION, document))\n .forEach(entity -> {\n entity.addEngine(Engine.NER);\n insert(entity);\n });\n end\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId)\n $entityToBeResized: RedactionEntity(matchesAnnotationId($id))\n then\n manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.setIgnored(true);\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.setIgnored(true);\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)\n $entityToForce: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToForce.apply(\"MAN.2.0\", \"Forced redaction\", $legalBasis);\n $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply image recategorization\"\n salience 128\n when\n ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)\n $image: Image($id == id)\n then\n $image.setImageType(ImageType.fromString($imageType));\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: RedactionEntity($type: type, $entityType: entityType)\n $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $contained.removeFromGraph();\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)\n $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $first.removeFromGraph();\n $second.removeFromGraph();\n RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);\n retract($first);\n retract($second);\n insert(mergedEntity);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)\n $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.removeFromGraph();\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)\n $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: RedactionEntity(entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $lowerRank.removeFromGraph();\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n" \ No newline at end of file +"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;\nimport com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualRedactionApplicationService manualRedactionApplicationService\nglobal NerEntitiesAdapter nerEntitiesAdapter\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//------------------------------------ Syngenta specific rules ------------------------------------\n\n// Rule unit: SYN.1\nrule \"SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n $section: Section(containsString(\"CT\") || containsString(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n entityCreationService.byRegexIgnoreCase(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.skip(\"SYN.1.0\", \"\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n\n\n//------------------------------------ CBI rules ------------------------------------\n\n// Rule unit: CBI.0\nrule \"CBI.0.0: Redact CBI Authors (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.0\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"CBI.0.1: Redact CBI Authors (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.1\nrule \"CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.skip(\"CBI.1.0\", \"Address found for Non Vertebrate Study\");\n end\n\nrule \"CBI.1.1: Redact CBI Address (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.apply(\"CBI.1.1\", \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.2\nrule \"CBI.2.0: Don't redact genitive CBI_author\"\n when\n $entity: RedactionEntity(type == \"CBI_author\", anyMatch(textAfter, \"['’’'ʼˈ´`‘′ʻ’']s\"), isApplied())\n then\n RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), \"CBI_author\", EntityType.FALSE_POSITIVE, document);\n falsePositive.skip(\"CBI.2.0\", \"Genitive Author found\");\n insert(falsePositive);\n end\n\n\n// Rule unit: CBI.7\nrule \"CBI.7.0: Do not redact Names and Addresses if published information found in section without tables\"\n when\n $section: Section(!hasTables(),\n hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $section.getEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.0\",\n \"Published Information found in section\",\n $section.getEntitiesOfType(\"published_information\")\n );\n });\n end\n\nrule \"CBI.7.1: Do not redact Names and Addresses if published information found in same table row\"\n when\n $table: Table(hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.1\",\n \"Published Information found in row\",\n $table.getEntitiesOfTypeInSameRow(\"published_information\", redactionEntity)\n );\n });\n end\n\n\n// Rule unit: CBI.9\nrule \"CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.0\", \"Author(s) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.1\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.10\nrule \"CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.0\", \"Author(s) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.11\nrule \"CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -1\n when\n $table: Table(hasHeader(\"Author(s)\") && hasHeader(\"Vertebrate Study Y/N\"))\n then\n $table.getEntitiesOfType(\"CBI_author\").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));\n end\n\n\n// Rule unit: CBI.16\nrule \"CBI.16.0: Add CBI_author with \\\"et al.\\\" Regex (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.0\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n insert(entity);\n });\n end\n\nrule \"CBI.16.1: Add CBI_author with \\\"et al.\\\" Regex (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.1\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n });\n end\n\n\n// Rule unit: CBI.17\nrule \"CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species\") && containsString(\"Source\") && !containsString(\"Species:\") && !containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.0\", \"Line after \\\"Source\\\" in Test Organism Section\");\n insert(entity);\n });\n end\n\nrule \"CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species:\"), containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source:\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.1\", \"Line after \\\"Source:\\\" in Test Animals Section\");\n insert(entity);\n });\n end\n\n\n// Rule unit: CBI.20\nrule \"CBI.20.0: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.skip(\"CBI.20.0\", \"PERFORMING LABORATORY was found for non vertebrate study\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\nrule \"CBI.20.1: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.apply(\"CBI.20.1\", \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\n\n//------------------------------------ PII rules ------------------------------------\n\n// Rule unit: PII.0\nrule \"PII.0.0: Redact all PII (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.0\", \"Personal Information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"PII.0.1: Redact all PII (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.1\", \"Personal Information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: PII.1\nrule \"PII.1.0: Redact Emails by RegEx (Non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.0\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\nrule \"PII.1.1: Redact Emails by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.1\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\n\n// Rule unit: PII.2\nrule \"PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.0\", \"Found by Phone and Fax Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\nrule \"PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.1\", \"Found by Phone and Fax Regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\n\n// Rule unit: PII.9\nrule \"PII.9.0: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.0\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.1: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.1\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.2: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.2\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.3: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.3\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\n\n//------------------------------------ Other rules ------------------------------------\n\n// Rule unit: ETC.0\nrule \"ETC.0.0: Purity Hint\"\n when\n $section: Section(containsStringIgnoreCase(\"purity\"))\n then\n entityCreationService.byRegexIgnoreCase(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", \"hint_only\", EntityType.ENTITY, 1, $section)\n .forEach(hint -> {\n hint.addEngine(Engine.RULE);\n hint.skip(\"ETC.0.0\", \"\");\n });\n end\n\n\n// Rule unit: ETC.2\nrule \"ETC.2.0: Redact signatures (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.2.0: Redact signatures (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.3\nrule \"ETC.3.0: Redact logos (vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.0\", \"Logo Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.3.1: Redact logos (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.1\", \"Logo Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.5\nrule \"ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n not FileAttribute(label == \"Confidentiality\", value == \"confidential\")\n $dossierRedaction: RedactionEntity(type == \"dossier_redaction\")\n then\n $dossierRedaction.removeFromGraph();\n retract($dossierRedaction);\n end\n\n\n//------------------------------------ AI rules ------------------------------------\n\n// Rule unit: AI.0\nrule \"AI.0.0: add all NER Entities of type CBI_author\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"CBI_author\"))\n then\n nerEntities.streamEntitiesOfType(\"CBI_author\")\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))\n .forEach(entity -> insert(entity));\n end\n\n\n// Rule unit: AI.1\nrule \"AI.1.0: combine and add NER Entities as CBI_address\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"ORG\") || hasEntitiesOfType(\"STREET\") || hasEntitiesOfType(\"CITY\"))\n then\n nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)\n .map(boundary -> entityCreationService.byBoundary(boundary, \"CBI_address\", EntityType.RECOMMENDATION, document))\n .forEach(entity -> {\n entity.addEngine(Engine.NER);\n insert(entity);\n });\n end\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId)\n $entityToBeResized: RedactionEntity(matchesAnnotationId($id))\n then\n manualRedactionApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.setIgnored(true);\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.setIgnored(true);\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)\n $entityToForce: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToForce.apply(\"MAN.2.0\", \"Forced redaction\", $legalBasis);\n $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply image recategorization\"\n salience 128\n when\n ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)\n $image: Image($id == id)\n then\n $image.setImageType(ImageType.fromString($imageType));\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: RedactionEntity($type: type, $entityType: entityType)\n $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $contained.removeFromGraph();\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)\n $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $first.removeFromGraph();\n $second.removeFromGraph();\n RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);\n retract($first);\n retract($second);\n insert(mergedEntity);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)\n $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.removeFromGraph();\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)\n $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: RedactionEntity(entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $lowerRank.removeFromGraph();\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt index d0051b4e..2020709c 100644 --- a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt +++ b/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt @@ -1 +1 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//---------------------------------------------------------------------------\n\n\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.0.1: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n\n then\n $tocHeadline.getParent().getPages()\n .forEach(page -> page.getMainBody().stream()\n .filter(node -> !node.getType().equals(NodeType.IMAGE))\n .filter(node -> node.getPages().stream().noneMatch(nodePage -> nodePage.getNumber() < page.getNumber()))\n .forEach(node -> retract(node))\n );\n end\n \n\n\n/*\nrule \"H.0.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.RECOMMENDATION);\n end\n*/\n\nrule \"DOC.0.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"408\",\"414\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n onPage(1)\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline, Method No. (\\\\d{3})\", 1, $section.getTextBlock()) \n ).flatMap(Collection::stream).findFirst()\n .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n onPage(1)\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// Examples found in PoC 1\n// entityCreationService.byRegex(\"((OECD Guidelines for Testing of Chemicals, Procedure)|(OECD Guidelines for the Testing of Chemicals No\\\\.)|(OECD Test Guideline)|(OECD \\\\[Test Guideline, Number)) \\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n// entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n// );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n// new approach OECD Guideline\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// missing OECD guideline rules for RFP demo file\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.1: Guidelines\"\n when\n $headline: Headline(\n onPage(1),\n containsString(\"OECD\")\n )\n then\n entityCreationService.byRegex(\"(OECD (No\\\\.? )?(\\\\d{3})( \\\\(\\\\d{4}\\\\))?)\", \"oecd_guideline\", EntityType.ENTITY,1, $headline).forEach(entity ->\n entity.apply(\"DOC.1.1\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\");\n });\n end\n\n\nrule \"DOC.1.3: Remove guidelines from irrelevant sections\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.remove(\"DOC.1.3\", \"remove guidelines from irrelevant sections\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.3.2: Experimental Completion Date\"\n salience 10\n when \n $section: Section(onPage(1) && (containsString(\"STUDY COMPLETED ON\") || containsString(\"STUDY COMPLETION DATE\") || containsString(\"Report completion date\") || containsString(\"Date of Report\") || containsString(\"AMENDMENT COMPLETION DATE\") || containsString(\"AMENDMENT COMPLETED ON\")))\n \n then\n entityCreationService.byRegex(\"STUDY COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"STUDY COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Report completion date (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Date of Report (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n end\n\n\n\n\n\n // hide all skipped species and strains except in the relevant sections\n rule \"DOC.4.2: Species\"\n salience 1\n when\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.remove(\"DOC.4.2\",\"n-a\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.4.3: Species\"\n when\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.4.3\", \"Species found.\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.0: Strain\"\n when\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.0\", \"Strain found.\");\n });\n end\n\n\n\n\nrule \"DOC.35.0: Sex\"\n when\n \n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.35.0\", \"Test animal sex found\");\n });\n end\n\n\n\nrule \"DOC.6.0: Authors\"\n when\n $headline: Headline(onPage(1), containsString(\"AUTHOR\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"author\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.6.0\", \"Author found\"));\n end\n\n\n\nrule \"DOC.6.2: Authors\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"AUTHOR(S)\" || contains \"AUTHORS\" || contains \"Author\"), getMainBodyTextBlock().getSearchText() (contains \"STUDY COMPLETED ON\" || contains \"STUDY COMPLETION DATE\" || contains \"DATE OF INTERIM REPORT\" || contains \"Report completion date\" || contains \"Date of Report\" || contains \"AMENDMENT COMPLETION DATE\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"AUTHOR(S)\", \"AUTHORS\", \"Author\"),\n List.of(\"STUDY COMPLETED ON\", \"STUDY COMPLETION DATE\", \"DATE OF INTERIM REPORT\", \"Report completion date\", \"Date of Report\", \"AMENDMENT COMPLETION DATE\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.2\", \"Author found\"));\n end\n\n\n\n\nrule \"DOC.6.6: laboratory_project_identification\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Laboratory Project Identification\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\"),\n List.of(\"SPONSOR\", \"VOLUME\", \"This\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.6\", \"Laboratory Project Identification\"));\n end\n\n\n\nrule \"DOC.7.2: study title by document structure\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"STUDY TITLE\" || contains \"Study Title\" || contains \"STUDYTITLE\" || contains \"Report (Final)\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"STUDY TITLE\", \"STUDYTITLE\", \"Report (Final)\"),\n List.of(\"TEST GUIDELINES\", \"TEST GUIDELINE(S)\", \"Guidelines\", \"DATA REQUIREMENT\", \"AUTHOR(S)\", \"AUTHOR\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.7.2\", \"Study title found\"));\n end\n\n\n\nrule \"DOC.8.1: Performing Laboratory\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"PERFORMING LABORATORY\" || contains \"TEST FACILITIES\" || contains \"TEST FACILITY\" || contains \"Test Facility\"), getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Sponsor\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"PERFORMING LABORATORY\", \"TEST FACILITIES\", \"TEST FACILITY\"),\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\", \"Sponsor\", \"PROJECT IDENTIFICATION\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.8.1\", \"Performing Laboratory found\"));\n end\n\n\n\n\nrule \"DOC.8.2: Summary Methods\"\n when\n $headline: Headline(containsString(\"1.1. METHODS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_methods\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.2\", \"Summary Methods found\"));\n end\n\nrule \"DOC.8.3: Summary Observations Laboratory\"\n when\n $headline: Headline(containsString(\"1.2. OBSERVATIONS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_observations\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.3\", \"Summary Observations found\"));\n end\n\n\nrule \"DOC.8.5: Summary Results\"\n when\n Headline((containsStringIgnoreCase(\"1.3. RESULTS\") || containsStringIgnoreCase(\"1.2. RESULTS\")), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> {\n entity.apply(\"DOC.8.5\", \"Summary Results\");\n });\n end\n\nrule \"DOC.8.6: Summary Results 2\"\n when\n $headline: Headline(containsString(\"1.2. RESULTS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\")) \n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.6\", \"Summary Results\"));\n end\n\n\n\nrule \"DOC.8.4: Summary Conclusion\"\n when\n $headline: Headline(containsString(\"1.4. CONCLUSION\") || containsString(\"1.3. CONCLUSION\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_conclusion\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.4\", \"Summary Conculsion found\"));\n end\n\n\n\nrule \"DOC.9.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\")\n || containsString(\"GLP CERTIFICATE (FROM THE COMPETENT AUTHORITY)\")\n || containsString(\"GLP COMPLIANCE STATEMENT\")\n || containsString(\"GLP STATEMENT\") \n )\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.9.0\", \"GLP Study found\");\n });\n end\n\n\n\nrule \"DOC.9.1: GLP Study\"\n when\n $paragraph: Paragraph(containsString(\"GLP COMPLIANCE STATEMENT\"))\n then\n entityCreationService.byRegex(\"GLP COMPLIANCE STATEMENT\", \"glp_study\", EntityType.ENTITY, $paragraph).forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"GLP Study found\");\n });\n end\n\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.getManualOverwrite().addChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: TextEntity($type: type, $entityType: entityType, !resized(), active())\n $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())\n then\n TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);\n $first.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n $second.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n retract($first);\n retract($second);\n mergedEntity.getIntersectingNodes().forEach(node -> update(node));\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())\n then\n $lowerRank.getIntersectingNodes().forEach(node -> update(node));\n $lowerRank.remove(\"X.6.0\", \"remove Entity of lower rank, when intersected by entity of type ENTITY\");\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file +"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//---------------------------------------------------------------------------\n\n\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.0.1: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n\n then\n $tocHeadline.getParent().getPages()\n .forEach(page -> page.getMainBody().stream()\n .filter(node -> !node.getType().equals(NodeType.IMAGE))\n .filter(node -> node.getPages().stream().noneMatch(nodePage -> nodePage.getNumber() < page.getNumber()))\n .forEach(node -> retract(node))\n );\n end\n \n\n\n/*\nrule \"H.0.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.RECOMMENDATION);\n end\n*/\n\nrule \"DOC.0.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"408\",\"414\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n onPage(1)\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline, Method No. (\\\\d{3})\", 1, $section.getTextBlock()) \n ).flatMap(Collection::stream).findFirst()\n .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n onPage(1)\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// Examples found in PoC 1\n// entityCreationService.byRegex(\"((OECD Guidelines for Testing of Chemicals, Procedure)|(OECD Guidelines for the Testing of Chemicals No\\\\.)|(OECD Test Guideline)|(OECD \\\\[Test Guideline, Number)) \\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n// entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n// );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n// new approach OECD Guideline\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// missing OECD guideline rules for RFP demo file\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.1: Guidelines\"\n when\n $headline: Headline(\n onPage(1),\n containsString(\"OECD\")\n )\n then\n entityCreationService.byRegex(\"(OECD (No\\\\.? )?(\\\\d{3})( \\\\(\\\\d{4}\\\\))?)\", \"oecd_guideline\", EntityType.ENTITY,1, $headline).forEach(entity ->\n entity.apply(\"DOC.1.1\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\");\n });\n end\n\n\nrule \"DOC.1.3: Remove guidelines from irrelevant sections\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.remove(\"DOC.1.3\", \"remove guidelines from irrelevant sections\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.3.2: Experimental Completion Date\"\n salience 10\n when \n $section: Section(onPage(1) && (containsString(\"STUDY COMPLETED ON\") || containsString(\"STUDY COMPLETION DATE\") || containsString(\"Report completion date\") || containsString(\"Date of Report\") || containsString(\"AMENDMENT COMPLETION DATE\") || containsString(\"AMENDMENT COMPLETED ON\")))\n \n then\n entityCreationService.byRegex(\"STUDY COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"STUDY COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Report completion date (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Date of Report (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n end\n\n\n\n\n\n // hide all skipped species and strains except in the relevant sections\n rule \"DOC.4.2: Species\"\n salience 1\n when\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.remove(\"DOC.4.2\",\"n-a\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.4.3: Species\"\n when\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.4.3\", \"Species found.\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.0: Strain\"\n when\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.0\", \"Strain found.\");\n });\n end\n\n\n\n\nrule \"DOC.35.0: Sex\"\n when\n \n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.35.0\", \"Test animal sex found\");\n });\n end\n\n\n\nrule \"DOC.6.0: Authors\"\n when\n $headline: Headline(onPage(1), containsString(\"AUTHOR\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"author\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.6.0\", \"Author found\"));\n end\n\n\n\nrule \"DOC.6.2: Authors\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"AUTHOR(S)\" || contains \"AUTHORS\" || contains \"Author\"), getMainBodyTextBlock().getSearchText() (contains \"STUDY COMPLETED ON\" || contains \"STUDY COMPLETION DATE\" || contains \"DATE OF INTERIM REPORT\" || contains \"Report completion date\" || contains \"Date of Report\" || contains \"AMENDMENT COMPLETION DATE\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"AUTHOR(S)\", \"AUTHORS\", \"Author\"),\n List.of(\"STUDY COMPLETED ON\", \"STUDY COMPLETION DATE\", \"DATE OF INTERIM REPORT\", \"Report completion date\", \"Date of Report\", \"AMENDMENT COMPLETION DATE\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.2\", \"Author found\"));\n end\n\n\n\n\nrule \"DOC.6.6: laboratory_project_identification\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Laboratory Project Identification\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\"),\n List.of(\"SPONSOR\", \"VOLUME\", \"This\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.6\", \"Laboratory Project Identification\"));\n end\n\n\n\nrule \"DOC.7.2: study title by document structure\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"STUDY TITLE\" || contains \"Study Title\" || contains \"STUDYTITLE\" || contains \"Report (Final)\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"STUDY TITLE\", \"STUDYTITLE\", \"Report (Final)\"),\n List.of(\"TEST GUIDELINES\", \"TEST GUIDELINE(S)\", \"Guidelines\", \"DATA REQUIREMENT\", \"AUTHOR(S)\", \"AUTHOR\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.7.2\", \"Study title found\"));\n end\n\n\n\nrule \"DOC.8.1: Performing Laboratory\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"PERFORMING LABORATORY\" || contains \"TEST FACILITIES\" || contains \"TEST FACILITY\" || contains \"Test Facility\"), getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Sponsor\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"PERFORMING LABORATORY\", \"TEST FACILITIES\", \"TEST FACILITY\"),\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\", \"Sponsor\", \"PROJECT IDENTIFICATION\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.8.1\", \"Performing Laboratory found\"));\n end\n\n\n\n\nrule \"DOC.8.2: Summary Methods\"\n when\n $headline: Headline(containsString(\"1.1. METHODS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_methods\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.2\", \"Summary Methods found\"));\n end\n\nrule \"DOC.8.3: Summary Observations Laboratory\"\n when\n $headline: Headline(containsString(\"1.2. OBSERVATIONS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_observations\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.3\", \"Summary Observations found\"));\n end\n\n\nrule \"DOC.8.5: Summary Results\"\n when\n Headline((containsStringIgnoreCase(\"1.3. RESULTS\") || containsStringIgnoreCase(\"1.2. RESULTS\")), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> {\n entity.apply(\"DOC.8.5\", \"Summary Results\");\n });\n end\n\nrule \"DOC.8.6: Summary Results 2\"\n when\n $headline: Headline(containsString(\"1.2. RESULTS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\")) \n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.6\", \"Summary Results\"));\n end\n\n\n\nrule \"DOC.8.4: Summary Conclusion\"\n when\n $headline: Headline(containsString(\"1.4. CONCLUSION\") || containsString(\"1.3. CONCLUSION\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_conclusion\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.4\", \"Summary Conculsion found\"));\n end\n\n\n\nrule \"DOC.9.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\")\n || containsString(\"GLP CERTIFICATE (FROM THE COMPETENT AUTHORITY)\")\n || containsString(\"GLP COMPLIANCE STATEMENT\")\n || containsString(\"GLP STATEMENT\") \n )\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.9.0\", \"GLP Study found\");\n });\n end\n\n\n\nrule \"DOC.9.1: GLP Study\"\n when\n $paragraph: Paragraph(containsString(\"GLP COMPLIANCE STATEMENT\"))\n then\n entityCreationService.byRegex(\"GLP COMPLIANCE STATEMENT\", \"glp_study\", EntityType.ENTITY, $paragraph).forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"GLP Study found\");\n });\n end\n\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.getManualOverwrite().addChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: TextEntity($type: type, $entityType: entityType, !resized(), active())\n $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())\n then\n TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);\n $first.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n $second.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n retract($first);\n retract($second);\n mergedEntity.getIntersectingNodes().forEach(node -> update(node));\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())\n then\n $lowerRank.getIntersectingNodes().forEach(node -> update(node));\n $lowerRank.remove(\"X.6.0\", \"remove Entity of lower rank, when intersected by entity of type ENTITY\");\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt index c1514d61..1739decc 100644 --- a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt +++ b/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt @@ -1 +1 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n//---------------------------------------------------------------------------\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.1.0: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n $page: Page() from $tocHeadline.getParent().getPages()\n $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))\n then\n retract($node);\n end\n\n\n// Rule unit: MAN.0\nrule \"H.2.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.ENTITY);\n end\n\n\nrule \"H.3.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n (containsString(\"DATA REQUIREMENT\") || containsString(\"TEST GUIDELINE\") || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\"))\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock())).flatMap(Collection::stream).findFirst()\n .map(textRange -> $section.getTextBlock().subSequence(textRange).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.1.3: Guidelines\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.2.0: Report number\"\n when\n $section: Section(containsString(\"LABORATORY PROJECT ID\") , containsString(\"Report Number:\"))\n then\n entityCreationService.lineAfterString(\"Report Number:\", \"report_number\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.2.0\", \"Report number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.3.0: Experimental Starting Date\"\n when\n $section: Section(containsString(\"Experimental I. Starting Date:\") || containsString(\"Experimental II. Starting Date:\") || containsStringIgnoreCase(\"experimental start date\") || containsStringIgnoreCase(\"experimental starting date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental start date\",\n \"Experimental start date:\",\n \"Experimental Starting Date\",\n \"Experimental Starting Date:\",\n \"Experimental starting date\",\n \"Experimental starting date:\",\n \"Experimental Start Date\",\n \"Experimental Start Date:\",\n \"Experimental I. Starting Date:\",\n \"Experimental II. Starting Date:\"), \"experimental_start_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.3.0\", \"Experimental start date found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.4.0: Experimental Completion Date\"\n when\n $section: Section(containsStringIgnoreCase(\"experimental termination date\") || containsStringIgnoreCase(\"experimental completion date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental termination date\",\n \"Experimental termination date:\",\n \"Experimental Completion Date\",\n \"Experimental Completion Date:\",\n \"Experimental completion date\",\n \"Experimental completion date:\",\n \"Experimental Termination Date\",\n \"Experimental Termination Date:\"), \"experimental_end_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.4.0\", \"Experimental end date found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.5.0: Ignore species and strain in irrelevant study types\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"471\",\"474\",\"487\"))\n $section: Section(hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\n rule \"DOC.5.1: Hide all skipped species and strains except in the relevant sections\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.5.2: Species\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.5.2\", \"Species found.\", \"n-a\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.3: Strain\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.3\", \"Strain found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.0: study title by document structure\"\n when\n $table: Table(onPage(1),\n (containsString(\"Final Report\") || containsString(\"SPL\")),\n numberOfRows == 1,\n numberOfCols == 1)\n then\n\n entityCreationService.bySemanticNode($table.getCell(0, 0).streamChildren().toList().get(1), \"title\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.6.0\", \"Study title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.1: study title\"\n when\n $table: Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $table).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.1\", \"Title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.2: study title\"\n when\n not Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n $section: Section(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.2\", \"Title found\", \"n-a\");\n });\n end\n\n\n\nrule \"DOC.7.0: Performing Laboratory (Name)\"\n when\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n entityCreationService.lineAfterString(\"PERFORMING LABORATORY:\", \"laboratory_name\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.7.0\", \"Performing Laboratory found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.7.1: Performing Laboratory (Country)\"\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"COUNTRY\"))\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n nerEntities.streamEntitiesOfType(\"COUNTRY\")\n .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, \"laboratory_country\", EntityType.ENTITY, $section))\n .forEach(entity -> {\n entity.apply(\"DOC.7.1\", \"Performing Laboratory found\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.7.2: Performing Laboratory (Country & Name) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && (containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(\"laboratory_country\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory country dictionary entry found.\", \"n-a\");\n });\n $section.getEntitiesOfType(\"laboratory_name\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory name dictionary entry found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.7.3: Performing Laboratory (Country) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && !(containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(List.of(\"laboratory_country\", \"laboratory_name\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.8.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\"))\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.8.0\", \"GLP Study found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.0: Batch number from CoA\"\n when\n $section: Section(\n (\n anyHeadlineContainsString(\"Analytical Report\")\n || anyHeadlineContainsStringIgnoreCase(\"Certificate of Analysis\")\n || containsStringIgnoreCase(\"Certificate of Analysis\")\n )\n && (\n containsStringIgnoreCase(\"batch\")\n || containsStringIgnoreCase(\"bath\")\n || containsStringIgnoreCase(\"barch\")\n || containsStringIgnoreCase(\"bateb\")\n )\n && (\n containsStringIgnoreCase(\"identification\")\n || containsStringIgnoreCase(\"ldentitfication\")\n || containsStringIgnoreCase(\"wentification\")\n || containsStringIgnoreCase(\"mentification\")\n || containsStringIgnoreCase(\"kientification\")\n || containsStringIgnoreCase(\"reference number\")\n || containsStringIgnoreCase(\"test substance\")\n )\n )\n then\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"(Batch Identification):\",\n \"Bateb Identification\",\n \"Batch Wentification\",\n \"Batch Mentification\",\n \"Batch Kientification\",\n \"Barch Identification\",\n \"Bath ldentitfication\",\n \"Batch of test substance :\"), \"batch_number\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.9.0\", \"Batch number found in CoA\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.1: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n then\n Stream.of(entityCreationService.byRegex(\"Batch ID ([A-Z\\\\d\\\\-]{7,14})\", \"batch_number\", EntityType.ENTITY, 1, $section),\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\"\n ), \"batch_number\", EntityType.ENTITY, $section)).flatMap(a -> a)\n .forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"Batch number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.2: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n $batchNumber: String() from List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\")\n $table: Table(containsStringIgnoreCase($batchNumber)) from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()\n then\n entityCreationService.lineAfterStringAcrossColumnsIgnoreCase($batchNumber, \"batch_number\", EntityType.ENTITY, $table).forEach(entity -> {\n entity.apply(\"DOC.9.2\", \"Batch number found\", \"n-a\");\n });\n end\n\n\n\n\nrule \"DOC.10.0: Conclusions - LD50, LC50, Confidence\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"425\",\"436\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"Conclusion\") || anyHeadlineContainsStringIgnoreCase(\"Lethality\"))\n && (containsString(\"LD\") || containsString(\"LC\") || containsString(\"50\") || containsString(\"LD50\") || containsString(\"lethal concentration\") || containsString(\"lethal dose\"))\n && (\n containsString(\"greater than\")\n || containsString(\"higher than\")\n || containsString(\"above\")\n || containsString(\"in excess\")\n || containsString(\"exceeds\")\n || containsString(\"was found to be\")\n || containsString(\"was calculated to be\")\n || containsString(\"estimated to be\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(L[D|C]\\\\s?50|lethal concentration|lethal dose).{1,200}(greater than|considered to be above|in excess of|exceeds|higher than)\", \"ld50_greater\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 greater than found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"\\\\b(?:(?:greater|higher) than|considered to be above|(?:was|is) (?:found|estimated) to be|was calculated to be|in excess of|exceeds|equal to)\\\\s?([\\\\d\\\\.]{1,6})\\\\s?mg\\\\/(?:kg|L)\", \"ld50_value\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 value found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_minimal\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Minimal Confidence found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_maximal\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Maximal Confidence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.0: Guideline Deviation\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"General Information\") || containsString(\"GENERAL INFORMATION\"))\n && (containsStringIgnoreCase(\"from the\") || containsStringIgnoreCase(\"to the\"))\n )\n then\n entityCreationService.betweenRegexes(\"(?:Deviations? from the [G|g]uidelines?)(?: and| or)?( the)?(?: Study Plan)?\", \"(?:(?:Deviations? from the Study Plan)|(?:Performing laboratory test)|(?:Other)|(?:Retention of [S|s]amples)|(?:Amendments? to Final Protocol))\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Guidelines found\", \"n-a\");\n });\n entityCreationService.betweenRegexes(\"(?:Deviations? (?:from|to)(?: the)? [S|s]tudy [P|p]lan)\", \"(?:Regulatory Guidelines)|(?:Other)|(?:Distribution of the report)|(?:Performing laboratory test)|(?:Distribution of the report)|(?:Retention of [S|s]amples)\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Study Plan found\", \"n-a\");\n });\n entityCreationService.betweenStrings(\"Deviations from the study plan\", \"Regulatory Guidelines\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?>Study plan adherence)(.{1,20}deviations.{1,20} to the study plan.{0,50}\\\\.)\\\\s\", \"guideline_deviation\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Guideline deviation found in text.\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeEnd(\"Deviations from the study plan\", \"validity of the study.\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.1: Guideline Deviation in text\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Introduction\")\n && containsStringIgnoreCase(\"deviations from the protocol\")\n )\n then\n entityCreationService.byRegex(\"There were no deviations from the protocol.{1,100}\\\\.\\\\s\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.1\", \"Guideline deviation found in text.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.12.0: Clinical Signs\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Clinical Signs\", \"Macroscopic Findings\") && !containsString(\"TABLE\") && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"clinical_signs\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.12.0\", \"Clinical Signs found\", \"n-a\"));\n end\n\n\nrule \"DOC.13.0: Dosages\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $section: Section(\n (anyHeadlineContainsStringIgnoreCase(\"Dosages\") || anyHeadlineContainsStringIgnoreCase(\"Study Design\"))\n && !getHeadline().containsString(\"TABLE\")\n )\n then\n entityCreationService.betweenStringsIncludeStartAndEnd(\"The animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeStartAndEnd(\"Animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.byRegexWithLineBreaks(\"(?:\\\\.[\\\\s|\\\\n]|^.{5,20}\\\\n)([^\\\\.]{1,200}(?:animal|given|received)[^\\\\.]{1,200}dose\\\\s(?:levels?\\\\s)?(?:of|at)[^\\\\.]{1,200})(?:\\\\.[\\\\s|\\\\n|$])\", \"dosages\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.14.0: Mortality\"\n when\n $headline: Headline(containsString(\"Mortality\") && !containsString(\"TABLE\"))\n FileAttribute(label == \"OECD Number\", value == \"425\")\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.14.0\", \"Mortality found\", \"n-a\"));\n end\n\n\nrule \"DOC.15.0: Study Conclusion\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Conclusion\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.15.0\", \"Study Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.16.0: Weight Behavior Changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && (\n containsString(\"body weight\")\n || containsString(\"body weights\")\n || containsString(\"bodyweight\")\n || containsString(\"bodyweights\")\n )\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"weight_behavior_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.16.0\", \"Weight behavior changes found\", \"n-a\"));\n end\n\n\nrule \"DOC.17.0: Necropsy findings\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"436\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Necropsy\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic Findings\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic examination\")\n )\n && !getHeadline().containsStringIgnoreCase(\"Table\")\n && !getHeadline().containsStringIgnoreCase(\"Appendix\")\n && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"necropsy_findings\", EntityType.ENTITY)\n .forEach( entity -> entity.apply(\"DOC.17.0\", \"Necropsy section found\", \"n-a\"));\n end\n\n\nrule \"DOC.18.0: Clinical observations\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Clinical Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Clinical observations\")\n || anyHeadlineContainsStringIgnoreCase(\"In-life Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Postmortem Observations\")\n )\n && !anyHeadlineContainsStringIgnoreCase(\"Appendix\")\n && !anyHeadlineContainsStringIgnoreCase(\"Table\")\n && !anyHeadlineContainsStringIgnoreCase(\"Mortality\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"clinical_observations\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.18.0\", \"Clinical observations section found\", \"n-a\"));\n end\n\n\nrule \"DOC.19.0: Bodyweight changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Bodyweight\", \"Bodyweights\", \"Body Weights\", \"Body Weight\"), !containsAnyStringIgnoreCase(\"Appendix\", \"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"bodyweight_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.19.0\", \"Bodyweight section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.0: Study Design\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"404\",\"405\",\"406\",\"428\",\"429\",\"438\",\"439\",\"474\",\"487\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n && !anyHeadlineContainsString(\"Preliminary screening test\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_design\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.20.0\", \"Study design section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.1: Study Design\"\n when\n Headline(containsStringIgnoreCase(\"Study Design\"), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_design\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.20.1\", \"Study design section found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.21.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n $parentHeadline: Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n not Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($parentHeadline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.0\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.21.1: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.1\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.22.0: Detailing (404 & 405)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"404\",\"405\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"Results\")\n && !getHeadline().containsStringIgnoreCase(\"Evaluation\")\n && !getHeadline().containsStringIgnoreCase(\"study\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"detailing\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.22.0\", \"Detailing found\", \"n-a\"));\n end\n\n\nrule \"DOC.23.0: Preliminary Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n ((anyHeadlineContainsString(\"Preliminary Screening Test\") && containsString(\"Clinical observations\"))\n || anyHeadlineContainsString(\"Pre-Experiment\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"preliminary_test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.23.0\", \"Preliminary Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.0: Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section((getHeadline().containsString(\"RESULTS AND DISCUSSION\") || getHeadline().containsString(\"Estimation of the proliferative response of lymph node cells\") || getHeadline().containsString(\"Results in the Main Experiment\")))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.24.0\", \"Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.1: Test Results (429)\"\n when\n Headline(containsStringIgnoreCase(\"RESULTS AND DISCUSSION\"), $sectionIdentifierResultsAndDiscussion: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifierResultsAndDiscussion))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"test_results\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.24.1\", \"Test Results found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.25.0: Approach used (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n hasEntitiesOfType(\"species\")\n && (containsStringIgnoreCase(\"animals per\") || containsStringIgnoreCase(\"animals /\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\banimals (?:per|\\\\/) .{0,15}(group)\\\\b\", \"approach_used\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.25.0\", \"Study animal approach found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.26.0: Sex\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"405\",\"429\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.26.0\", \"Test animal sex found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.27.0: Animal Number 405\"\n when\n FileAttribute(label == \"OECD Number\", value == \"405\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"reaction\")\n )\n && !getHeadline().containsString(\"selection\")\n && (\n containsStringIgnoreCase(\"number of animals\")\n || containsStringIgnoreCase(\"no.\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?:.{1,10} No\\\\. )([\\\\d\\\\w\\\\-]{3,8})\", \"animal_numbers\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.0: Animal Number 429\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsString(\"selection\")\n && containsStringIgnoreCase(\"number of animals\")\n && (containsStringIgnoreCase(\"per\") || containsString(\"/\"))\n && containsStringIgnoreCase(\"group\")\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:).{0,60}\\\\b([\\\\d]{1,3})\\\\sper group\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"([\\\\d]{1,3})[\\\\w\\\\s\\\\/]{0,20}(?:treatment )?group\\\\b\", \"number_of_animals\", EntityType.ENTITY,1 , $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.1: No. Of animals - Fallback to appendix tables listing all individual animals for 429\"\n when\n $keyword: String() from List.of(\"Animal Number\", \"Animal No.\", \"Animal number\")\n $table: Table(containsString($keyword) && getHeadline().containsString(\"TABLE\") && getHeadline().containsString(\"Individual\"))\n FileAttribute(label == \"OECD Number\", value == \"429\")\n then\n $table.streamTableCellsWithHeader($keyword)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"animal_numbers\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.28.1\", \"Animal number found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.29.0: 4h Exposure\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"403\",\"436\"))\n $section: Section(\n (containsStringIgnoreCase(\"4 hours\") || containsStringIgnoreCase(\"four hours\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"((?<=\\\\.\\\\s\\\\b).{1,100}(4|four) hours.*?\\\\.) \", \"4h_exposure\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.29.0\", \"4h exposure sentence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.30.0: Dilution of the test substance\"\n when\n FileAttribute(label == \"OECD Number\", value == \"404\")\n $section: Section(\n getHeadline().containsString(\"Formulation\")\n && containsString(\"dilution\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"dilution\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.30.0\", \"Dilution found.\", \"n-a\"));\n end\n\n\nrule \"DOC.31.0: Positive Control\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Positive Control\")\n && !(getHeadline().containsStringIgnoreCase(\"Appendix\") || getHeadline().containsStringIgnoreCase(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"positive_control\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.31.0\", \"Positive control found.\", \"n-a\"));\n end\n\n\nrule \"DOC.32.0: Mortality Statement\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $headline: Headline(containsStringIgnoreCase(\"Mortality\") && !containsString(\"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality_statement\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.32.0\", \"Mortality Statement found\", \"n-a\"));\n end\n\n\nrule \"DOC.33.0: Dose Mortality\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $table: Table(\n (hasHeader(\"Mortality\") || hasHeader(\"Long Term Results\") || hasHeader(\"LongTerm Outcome\") || hasHeader(\"Long Term Outcome\") || hasHeader(\"Comments\") || hasHeader(\"Viability / Mortality\") || hasHeader(\"Viability/Mortality\"))\n &&\n (hasHeader(\"Dose [mg/kg bodyweight]\") || hasHeader(\"Dose [mg/kg body weight]\") ||hasHeader(\"Dose (mg/kg)\") || hasHeader(\"Dose levei (mg/kg)\") || hasHeader(\"Dose Level (mg/kg)\") || hasHeader(\"Dose level (mg/kg)\") || hasHeader(\"Dosage [mg/kg body weight]\"))\n )\n then\n Stream.of($table.streamTableCellsWithHeader(\"Mortality\"),\n $table.streamTableCellsWithHeader(\"Comments\"),\n $table.streamTableCellsWithHeader(\"Long Term Results\"),\n $table.streamTableCellsWithHeader(\"Long Term Outcome\"),\n $table.streamTableCellsWithHeader(\"LongTerm Outcome\"),\n $table.streamTableCellsWithHeader(\"Viability / Mortality\"),\n $table.streamTableCellsWithHeader(\"Viability/Mortality\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n\n Stream.of($table.streamTableCellsWithHeader(\"Dose [mg/kg bodyweight]\"),\n $table.streamTableCellsWithHeader(\"Dose [mg/kg body weight]\"),\n $table.streamTableCellsWithHeader(\"Dose levei (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose Level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dosage [mg/kg body weight]\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality_dose\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.34.0: Results (Main Study)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && getHeadline().getTextRange().length() < 20\n && !(getHeadline().containsString(\"Appendix\") || getHeadline().containsString(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"results_(main_study)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.34.0\", \"Results for main study found.\", \"n-a\"));\n end\n\n\nrule \"DOC.35.0: Doses (mg/kg bodyweight)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"doses_(mg_kg_bw)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.35.0\", \"Doses per bodyweight information found\", \"n-a\"));\n end\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.getManualOverwrite().addChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);\n end\n\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.7\nrule \"X.7.0: remove all images\"\n salience 512\n when\n $image: Image(imageType != ImageType.OCR, !hasManualChanges())\n then\n $image.remove(\"X.7.0\", \"remove all images\");\n retract($image);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file +"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n//---------------------------------------------------------------------------\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.1.0: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n $page: Page() from $tocHeadline.getParent().getPages()\n $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))\n then\n retract($node);\n end\n\n\n// Rule unit: MAN.0\nrule \"H.2.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.ENTITY);\n end\n\n\nrule \"H.3.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n (containsString(\"DATA REQUIREMENT\") || containsString(\"TEST GUIDELINE\") || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\"))\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock())).flatMap(Collection::stream).findFirst()\n .map(textRange -> $section.getTextBlock().subSequence(textRange).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.1.3: Guidelines\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.2.0: Report number\"\n when\n $section: Section(containsString(\"LABORATORY PROJECT ID\") , containsString(\"Report Number:\"))\n then\n entityCreationService.lineAfterString(\"Report Number:\", \"report_number\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.2.0\", \"Report number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.3.0: Experimental Starting Date\"\n when\n $section: Section(containsString(\"Experimental I. Starting Date:\") || containsString(\"Experimental II. Starting Date:\") || containsStringIgnoreCase(\"experimental start date\") || containsStringIgnoreCase(\"experimental starting date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental start date\",\n \"Experimental start date:\",\n \"Experimental Starting Date\",\n \"Experimental Starting Date:\",\n \"Experimental starting date\",\n \"Experimental starting date:\",\n \"Experimental Start Date\",\n \"Experimental Start Date:\",\n \"Experimental I. Starting Date:\",\n \"Experimental II. Starting Date:\"), \"experimental_start_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.3.0\", \"Experimental start date found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.4.0: Experimental Completion Date\"\n when\n $section: Section(containsStringIgnoreCase(\"experimental termination date\") || containsStringIgnoreCase(\"experimental completion date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental termination date\",\n \"Experimental termination date:\",\n \"Experimental Completion Date\",\n \"Experimental Completion Date:\",\n \"Experimental completion date\",\n \"Experimental completion date:\",\n \"Experimental Termination Date\",\n \"Experimental Termination Date:\"), \"experimental_end_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.4.0\", \"Experimental end date found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.5.0: Ignore species and strain in irrelevant study types\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"471\",\"474\",\"487\"))\n $section: Section(hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\n rule \"DOC.5.1: Hide all skipped species and strains except in the relevant sections\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.5.2: Species\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.5.2\", \"Species found.\", \"n-a\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.3: Strain\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.3\", \"Strain found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.0: study title by document structure\"\n when\n $table: Table(onPage(1),\n (containsString(\"Final Report\") || containsString(\"SPL\")),\n numberOfRows == 1,\n numberOfCols == 1)\n then\n\n entityCreationService.bySemanticNode($table.getCell(0, 0).streamChildren().toList().get(1), \"title\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.6.0\", \"Study title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.1: study title\"\n when\n $table: Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $table).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.1\", \"Title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.2: study title\"\n when\n not Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n $section: Section(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.2\", \"Title found\", \"n-a\");\n });\n end\n\n\n\nrule \"DOC.7.0: Performing Laboratory (Name)\"\n when\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n entityCreationService.lineAfterString(\"PERFORMING LABORATORY:\", \"laboratory_name\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.7.0\", \"Performing Laboratory found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.7.1: Performing Laboratory (Country)\"\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"COUNTRY\"))\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n nerEntities.streamEntitiesOfType(\"COUNTRY\")\n .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, \"laboratory_country\", EntityType.ENTITY, $section))\n .forEach(entity -> {\n entity.apply(\"DOC.7.1\", \"Performing Laboratory found\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.7.2: Performing Laboratory (Country & Name) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && (containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(\"laboratory_country\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory country dictionary entry found.\", \"n-a\");\n });\n $section.getEntitiesOfType(\"laboratory_name\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory name dictionary entry found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.7.3: Performing Laboratory (Country) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && !(containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(List.of(\"laboratory_country\", \"laboratory_name\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.8.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\"))\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.8.0\", \"GLP Study found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.0: Batch number from CoA\"\n when\n $section: Section(\n (\n anyHeadlineContainsString(\"Analytical Report\")\n || anyHeadlineContainsStringIgnoreCase(\"Certificate of Analysis\")\n || containsStringIgnoreCase(\"Certificate of Analysis\")\n )\n && (\n containsStringIgnoreCase(\"batch\")\n || containsStringIgnoreCase(\"bath\")\n || containsStringIgnoreCase(\"barch\")\n || containsStringIgnoreCase(\"bateb\")\n )\n && (\n containsStringIgnoreCase(\"identification\")\n || containsStringIgnoreCase(\"ldentitfication\")\n || containsStringIgnoreCase(\"wentification\")\n || containsStringIgnoreCase(\"mentification\")\n || containsStringIgnoreCase(\"kientification\")\n || containsStringIgnoreCase(\"reference number\")\n || containsStringIgnoreCase(\"test substance\")\n )\n )\n then\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"(Batch Identification):\",\n \"Bateb Identification\",\n \"Batch Wentification\",\n \"Batch Mentification\",\n \"Batch Kientification\",\n \"Barch Identification\",\n \"Bath ldentitfication\",\n \"Batch of test substance :\"), \"batch_number\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.9.0\", \"Batch number found in CoA\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.1: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n then\n Stream.of(entityCreationService.byRegex(\"Batch ID ([A-Z\\\\d\\\\-]{7,14})\", \"batch_number\", EntityType.ENTITY, 1, $section),\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\"\n ), \"batch_number\", EntityType.ENTITY, $section)).flatMap(a -> a)\n .forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"Batch number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.2: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n $batchNumber: String() from List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\")\n $table: Table(containsStringIgnoreCase($batchNumber)) from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()\n then\n entityCreationService.lineAfterStringAcrossColumnsIgnoreCase($batchNumber, \"batch_number\", EntityType.ENTITY, $table).forEach(entity -> {\n entity.apply(\"DOC.9.2\", \"Batch number found\", \"n-a\");\n });\n end\n\n\n\n\nrule \"DOC.10.0: Conclusions - LD50, LC50, Confidence\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"425\",\"436\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"Conclusion\") || anyHeadlineContainsStringIgnoreCase(\"Lethality\"))\n && (containsString(\"LD\") || containsString(\"LC\") || containsString(\"50\") || containsString(\"LD50\") || containsString(\"lethal concentration\") || containsString(\"lethal dose\"))\n && (\n containsString(\"greater than\")\n || containsString(\"higher than\")\n || containsString(\"above\")\n || containsString(\"in excess\")\n || containsString(\"exceeds\")\n || containsString(\"was found to be\")\n || containsString(\"was calculated to be\")\n || containsString(\"estimated to be\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(L[D|C]\\\\s?50|lethal concentration|lethal dose).{1,200}(greater than|considered to be above|in excess of|exceeds|higher than)\", \"ld50_greater\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 greater than found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"\\\\b(?:(?:greater|higher) than|considered to be above|(?:was|is) (?:found|estimated) to be|was calculated to be|in excess of|exceeds|equal to)\\\\s?([\\\\d\\\\.]{1,6})\\\\s?mg\\\\/(?:kg|L)\", \"ld50_value\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 value found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_minimal\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Minimal Confidence found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_maximal\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Maximal Confidence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.0: Guideline Deviation\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"General Information\") || containsString(\"GENERAL INFORMATION\"))\n && (containsStringIgnoreCase(\"from the\") || containsStringIgnoreCase(\"to the\"))\n )\n then\n entityCreationService.betweenRegexes(\"(?:Deviations? from the [G|g]uidelines?)(?: and| or)?( the)?(?: Study Plan)?\", \"(?:(?:Deviations? from the Study Plan)|(?:Performing laboratory test)|(?:Other)|(?:Retention of [S|s]amples)|(?:Amendments? to Final Protocol))\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Guidelines found\", \"n-a\");\n });\n entityCreationService.betweenRegexes(\"(?:Deviations? (?:from|to)(?: the)? [S|s]tudy [P|p]lan)\", \"(?:Regulatory Guidelines)|(?:Other)|(?:Distribution of the report)|(?:Performing laboratory test)|(?:Distribution of the report)|(?:Retention of [S|s]amples)\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Study Plan found\", \"n-a\");\n });\n entityCreationService.betweenStrings(\"Deviations from the study plan\", \"Regulatory Guidelines\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?>Study plan adherence)(.{1,20}deviations.{1,20} to the study plan.{0,50}\\\\.)\\\\s\", \"guideline_deviation\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Guideline deviation found in text.\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeEnd(\"Deviations from the study plan\", \"validity of the study.\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.1: Guideline Deviation in text\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Introduction\")\n && containsStringIgnoreCase(\"deviations from the protocol\")\n )\n then\n entityCreationService.byRegex(\"There were no deviations from the protocol.{1,100}\\\\.\\\\s\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.1\", \"Guideline deviation found in text.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.12.0: Clinical Signs\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Clinical Signs\", \"Macroscopic Findings\") && !containsString(\"TABLE\") && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"clinical_signs\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.12.0\", \"Clinical Signs found\", \"n-a\"));\n end\n\n\nrule \"DOC.13.0: Dosages\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $section: Section(\n (anyHeadlineContainsStringIgnoreCase(\"Dosages\") || anyHeadlineContainsStringIgnoreCase(\"Study Design\"))\n && !getHeadline().containsString(\"TABLE\")\n )\n then\n entityCreationService.betweenStringsIncludeStartAndEnd(\"The animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeStartAndEnd(\"Animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.byRegexWithLineBreaks(\"(?:\\\\.[\\\\s|\\\\n]|^.{5,20}\\\\n)([^\\\\.]{1,200}(?:animal|given|received)[^\\\\.]{1,200}dose\\\\s(?:levels?\\\\s)?(?:of|at)[^\\\\.]{1,200})(?:\\\\.[\\\\s|\\\\n|$])\", \"dosages\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.14.0: Mortality\"\n when\n $headline: Headline(containsString(\"Mortality\") && !containsString(\"TABLE\"))\n FileAttribute(label == \"OECD Number\", value == \"425\")\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.14.0\", \"Mortality found\", \"n-a\"));\n end\n\n\nrule \"DOC.15.0: Study Conclusion\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Conclusion\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.15.0\", \"Study Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.16.0: Weight Behavior Changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && (\n containsString(\"body weight\")\n || containsString(\"body weights\")\n || containsString(\"bodyweight\")\n || containsString(\"bodyweights\")\n )\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"weight_behavior_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.16.0\", \"Weight behavior changes found\", \"n-a\"));\n end\n\n\nrule \"DOC.17.0: Necropsy findings\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"436\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Necropsy\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic Findings\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic examination\")\n )\n && !getHeadline().containsStringIgnoreCase(\"Table\")\n && !getHeadline().containsStringIgnoreCase(\"Appendix\")\n && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"necropsy_findings\", EntityType.ENTITY)\n .forEach( entity -> entity.apply(\"DOC.17.0\", \"Necropsy section found\", \"n-a\"));\n end\n\n\nrule \"DOC.18.0: Clinical observations\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Clinical Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Clinical observations\")\n || anyHeadlineContainsStringIgnoreCase(\"In-life Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Postmortem Observations\")\n )\n && !anyHeadlineContainsStringIgnoreCase(\"Appendix\")\n && !anyHeadlineContainsStringIgnoreCase(\"Table\")\n && !anyHeadlineContainsStringIgnoreCase(\"Mortality\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"clinical_observations\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.18.0\", \"Clinical observations section found\", \"n-a\"));\n end\n\n\nrule \"DOC.19.0: Bodyweight changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Bodyweight\", \"Bodyweights\", \"Body Weights\", \"Body Weight\"), !containsAnyStringIgnoreCase(\"Appendix\", \"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"bodyweight_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.19.0\", \"Bodyweight section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.0: Study Design\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"404\",\"405\",\"406\",\"428\",\"429\",\"438\",\"439\",\"474\",\"487\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n && !anyHeadlineContainsString(\"Preliminary screening test\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_design\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.20.0\", \"Study design section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.1: Study Design\"\n when\n Headline(containsStringIgnoreCase(\"Study Design\"), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_design\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.20.1\", \"Study design section found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.21.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n $parentHeadline: Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n not Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($parentHeadline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.0\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.21.1: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.1\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.22.0: Detailing (404 & 405)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"404\",\"405\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"Results\")\n && !getHeadline().containsStringIgnoreCase(\"Evaluation\")\n && !getHeadline().containsStringIgnoreCase(\"study\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"detailing\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.22.0\", \"Detailing found\", \"n-a\"));\n end\n\n\nrule \"DOC.23.0: Preliminary Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n ((anyHeadlineContainsString(\"Preliminary Screening Test\") && containsString(\"Clinical observations\"))\n || anyHeadlineContainsString(\"Pre-Experiment\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"preliminary_test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.23.0\", \"Preliminary Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.0: Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section((getHeadline().containsString(\"RESULTS AND DISCUSSION\") || getHeadline().containsString(\"Estimation of the proliferative response of lymph node cells\") || getHeadline().containsString(\"Results in the Main Experiment\")))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.24.0\", \"Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.1: Test Results (429)\"\n when\n Headline(containsStringIgnoreCase(\"RESULTS AND DISCUSSION\"), $sectionIdentifierResultsAndDiscussion: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifierResultsAndDiscussion))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"test_results\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.24.1\", \"Test Results found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.25.0: Approach used (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n hasEntitiesOfType(\"species\")\n && (containsStringIgnoreCase(\"animals per\") || containsStringIgnoreCase(\"animals /\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\banimals (?:per|\\\\/) .{0,15}(group)\\\\b\", \"approach_used\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.25.0\", \"Study animal approach found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.26.0: Sex\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"405\",\"429\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.26.0\", \"Test animal sex found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.27.0: Animal Number 405\"\n when\n FileAttribute(label == \"OECD Number\", value == \"405\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"reaction\")\n )\n && !getHeadline().containsString(\"selection\")\n && (\n containsStringIgnoreCase(\"number of animals\")\n || containsStringIgnoreCase(\"no.\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?:.{1,10} No\\\\. )([\\\\d\\\\w\\\\-]{3,8})\", \"animal_numbers\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.0: Animal Number 429\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsString(\"selection\")\n && containsStringIgnoreCase(\"number of animals\")\n && (containsStringIgnoreCase(\"per\") || containsString(\"/\"))\n && containsStringIgnoreCase(\"group\")\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:).{0,60}\\\\b([\\\\d]{1,3})\\\\sper group\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"([\\\\d]{1,3})[\\\\w\\\\s\\\\/]{0,20}(?:treatment )?group\\\\b\", \"number_of_animals\", EntityType.ENTITY,1 , $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.1: No. Of animals - Fallback to appendix tables listing all individual animals for 429\"\n when\n $keyword: String() from List.of(\"Animal Number\", \"Animal No.\", \"Animal number\")\n $table: Table(containsString($keyword) && getHeadline().containsString(\"TABLE\") && getHeadline().containsString(\"Individual\"))\n FileAttribute(label == \"OECD Number\", value == \"429\")\n then\n $table.streamTableCellsWithHeader($keyword)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"animal_numbers\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.28.1\", \"Animal number found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.29.0: 4h Exposure\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"403\",\"436\"))\n $section: Section(\n (containsStringIgnoreCase(\"4 hours\") || containsStringIgnoreCase(\"four hours\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"((?<=\\\\.\\\\s\\\\b).{1,100}(4|four) hours.*?\\\\.) \", \"4h_exposure\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.29.0\", \"4h exposure sentence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.30.0: Dilution of the test substance\"\n when\n FileAttribute(label == \"OECD Number\", value == \"404\")\n $section: Section(\n getHeadline().containsString(\"Formulation\")\n && containsString(\"dilution\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"dilution\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.30.0\", \"Dilution found.\", \"n-a\"));\n end\n\n\nrule \"DOC.31.0: Positive Control\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Positive Control\")\n && !(getHeadline().containsStringIgnoreCase(\"Appendix\") || getHeadline().containsStringIgnoreCase(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"positive_control\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.31.0\", \"Positive control found.\", \"n-a\"));\n end\n\n\nrule \"DOC.32.0: Mortality Statement\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $headline: Headline(containsStringIgnoreCase(\"Mortality\") && !containsString(\"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality_statement\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.32.0\", \"Mortality Statement found\", \"n-a\"));\n end\n\n\nrule \"DOC.33.0: Dose Mortality\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $table: Table(\n (hasHeader(\"Mortality\") || hasHeader(\"Long Term Results\") || hasHeader(\"LongTerm Outcome\") || hasHeader(\"Long Term Outcome\") || hasHeader(\"Comments\") || hasHeader(\"Viability / Mortality\") || hasHeader(\"Viability/Mortality\"))\n &&\n (hasHeader(\"Dose [mg/kg bodyweight]\") || hasHeader(\"Dose [mg/kg body weight]\") ||hasHeader(\"Dose (mg/kg)\") || hasHeader(\"Dose levei (mg/kg)\") || hasHeader(\"Dose Level (mg/kg)\") || hasHeader(\"Dose level (mg/kg)\") || hasHeader(\"Dosage [mg/kg body weight]\"))\n )\n then\n Stream.of($table.streamTableCellsWithHeader(\"Mortality\"),\n $table.streamTableCellsWithHeader(\"Comments\"),\n $table.streamTableCellsWithHeader(\"Long Term Results\"),\n $table.streamTableCellsWithHeader(\"Long Term Outcome\"),\n $table.streamTableCellsWithHeader(\"LongTerm Outcome\"),\n $table.streamTableCellsWithHeader(\"Viability / Mortality\"),\n $table.streamTableCellsWithHeader(\"Viability/Mortality\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n\n Stream.of($table.streamTableCellsWithHeader(\"Dose [mg/kg bodyweight]\"),\n $table.streamTableCellsWithHeader(\"Dose [mg/kg body weight]\"),\n $table.streamTableCellsWithHeader(\"Dose levei (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose Level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dosage [mg/kg body weight]\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality_dose\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.34.0: Results (Main Study)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && getHeadline().getTextRange().length() < 20\n && !(getHeadline().containsString(\"Appendix\") || getHeadline().containsString(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"results_(main_study)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.34.0\", \"Results for main study found.\", \"n-a\"));\n end\n\n\nrule \"DOC.35.0: Doses (mg/kg bodyweight)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"doses_(mg_kg_bw)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.35.0\", \"Doses per bodyweight information found\", \"n-a\"));\n end\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.getManualOverwrite().addChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);\n end\n\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.7\nrule \"X.7.0: remove all images\"\n salience 512\n when\n $image: Image(imageType != ImageType.OCR, !hasManualChanges())\n then\n $image.remove(\"X.7.0\", \"remove all images\");\n retract($image);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file