diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java index 3e990759..21baf0cf 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java @@ -9,6 +9,10 @@ import java.util.List; import java.util.Optional; import java.util.stream.Stream; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.GenericSemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; @@ -17,6 +21,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.utils.EntityCreationUtility; +import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -358,6 +364,36 @@ public class DocumentTree { } + public boolean addEntityToGraph(TextEntity entity) { + + getRoot().getNode().addThisToEntityIfIntersects(entity); + + TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock(); + EntityEnrichmentService.enrichEntity(entity, textBlock); + + EntityCreationUtility.addToPages(entity); + EntityCreationUtility.addEntityToNodeEntitySets(entity); + for (TextEntity textEntity : entity.getDeepestFullyContainingNode().getEntities()) { + if (entity.intersects(textEntity) && !entity.equals(textEntity)) { + if (textEntity.getTextRange().equals(entity.getTextRange())) { + textEntity.getRelations().put(textEntity, new Equality(entity, textEntity)); + entity.getRelations().put(textEntity, new Equality(textEntity, entity)); + } else if (textEntity.containedBy(entity)) { + entity.getRelations().put(textEntity, new Containment(entity, textEntity)); + textEntity.getRelations().put(entity, new Intersection(textEntity, entity)); + } else if (entity.containedBy(textEntity)) { + textEntity.getRelations().put(entity, new Containment(textEntity, entity)); + entity.getRelations().put(textEntity, new Intersection(entity, textEntity)); + } else { + entity.getRelations().put(textEntity, new Intersection(entity, textEntity)); + textEntity.getRelations().put(entity, new Intersection(textEntity, entity)); + } + } + } + return true; + } + + @Builder @Getter @AllArgsConstructor diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/IntersectingNodeVisitor.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/IntersectingNodeVisitor.java new file mode 100644 index 00000000..d0ec319e --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/IntersectingNodeVisitor.java @@ -0,0 +1,32 @@ +package com.iqser.red.service.redaction.v1.server.model.document; + +import java.util.HashSet; +import java.util.Set; + +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; + +import lombok.Getter; + +public class IntersectingNodeVisitor extends AbstractNodeVisitor { + + @Getter + private Set intersectingNodes; + private final TextRange textRange; + + + public IntersectingNodeVisitor(TextRange textRange) { + + this.textRange = textRange; + this.intersectingNodes = new HashSet<>(); + } + + + @Override + public void visitNodeDefault(SemanticNode node) { + + if (textRange.intersects(node.getTextRange())) { + intersectingNodes.add(node); + } + } + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractBidirectionalRelation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractBidirectionalRelation.java deleted file mode 100644 index 55aae869..00000000 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractBidirectionalRelation.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.model.document.entity; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -@Getter -@RequiredArgsConstructor -public abstract class AbstractBidirectionalRelation implements BidirectionalRelation { - - protected final TextEntity a; - protected final TextEntity b; - - @Override - public abstract BidirectionalRelation invert(); - -} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/BidirectionalRelation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/BidirectionalRelation.java deleted file mode 100644 index 57b70509..00000000 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/BidirectionalRelation.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.model.document.entity; - -public interface BidirectionalRelation extends Relation { - - BidirectionalRelation invert(); - -} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java index 2d621959..d3ac61f1 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java @@ -1,10 +1,18 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity; -public class Containment extends AbstractRelation { +public class Containment extends Intersection { public Containment(TextEntity container, TextEntity contained) { super(container, contained); } + public TextEntity getContainer() { + return a; + } + + public TextEntity getContained() { + return b; + } + } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java index 1550265e..3af87e67 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java @@ -6,5 +6,6 @@ public enum EntityType { RECOMMENDATION, FALSE_POSITIVE, FALSE_RECOMMENDATION, - DICTIONARY_REMOVAL + DICTIONARY_REMOVAL, + TEMPORARY } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java index 5852ed50..9dc80670 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java @@ -1,17 +1,10 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity; -public class Equality extends AbstractBidirectionalRelation { +public class Equality extends Containment { public Equality(TextEntity a, TextEntity b) { super(a, b); } - - @Override - public Equality invert() { - - return new Equality(b, a); - } - } \ No newline at end of file diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java index 65c9dea2..a124bc14 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java @@ -51,16 +51,18 @@ public interface IEntity { String type(); - /** * Marks this entity and all its intersecting nodes as updated */ void update(); + + /** * Marks this entity and all its intersecting nodes as removed */ void remove(); + /** * An Entity is valid, when it active and not a false recommendation, a false positive or a dictionary removal. * @@ -349,12 +351,18 @@ public interface IEntity { */ default void addMatchedRule(MatchedRule matchedRule) { - boolean valid = valid(); + boolean wasValid = valid(); getMatchedRuleList().add(matchedRule); - if (valid() == valid) { + handleStateChange(wasValid); + } + + + default void handleStateChange(boolean wasValid) { + + if (valid() == wasValid) { return; } - if(!removed()) { + if (!removed()) { update(); } else { remove(); diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java index 00e57fe5..f31f2dcd 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java @@ -1,17 +1,10 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity; -public class Intersection extends AbstractBidirectionalRelation { +public class Intersection extends AbstractRelation { public Intersection(TextEntity a, TextEntity b) { super(a, b); } - - @Override - public Intersection invert() { - - return new Intersection(b, a); - } - } \ No newline at end of file diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java index cfbe1662..5ee4c4dc 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java @@ -6,4 +6,5 @@ public interface Relation { TextEntity getB(); + } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RelationStore.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RelationStore.java deleted file mode 100644 index 296d632e..00000000 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RelationStore.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.model.document.entity; - -import java.util.*; -import java.util.stream.Collectors; - -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.stereotype.Service; - -@Service -public class RelationStore { - - // -// private final Map, Relation> relations = new HashMap<>(); - private final Map> entityRelations = new HashMap<>(); - - - public void addRelation(Relation relation) { - - String idA = relation.getA().getId(); - String idB = relation.getB().getId(); - - Pair key = Pair.of(idA, idB); -// relations.put(key, relation); - - if (relation instanceof BidirectionalRelation bidirectionalRelation) { - -// BidirectionalRelation invertedRelation = bidirectionalRelation.invert(); -// Pair invertedKey = Pair.of(idB, idA); -// relations.put(invertedKey, invertedRelation); - entityRelations.computeIfAbsent(idB, k -> new HashSet<>()).add(relation); - } - - entityRelations.computeIfAbsent(idA, k -> new HashSet<>()).add(relation); - } - - - public void removeRelation(Relation relation) { - - String idA = relation.getA().getId(); - String idB = relation.getB().getId(); - - Pair key = Pair.of(idA, idB); -// relations.remove(key); - - if (relation instanceof BidirectionalRelation) { - -// Pair invertedKey = Pair.of(idB, idA); -// relations.remove(invertedKey); - entityRelations.getOrDefault(idB, Collections.emptySet()).remove(relation); - } - - entityRelations.getOrDefault(idA, Collections.emptySet()).remove(relation); - } - - - public Set getRelations(String entityId) { - - return entityRelations.getOrDefault(entityId, Collections.emptySet()); - } - -// -// public Relation getRelation(String idA, String idB) { -// -// return relations.get(Pair.of(idA, idB)); -// } - - - public Set getAllRelations(Set entityIds) { - - return entityRelations.entrySet() - .stream() - .filter(entry -> entityIds.contains(entry.getKey())) - .flatMap(entry -> entry.getValue() - .stream()) - .collect(Collectors.toSet()); - } - -} - diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index dbdbbf46..8cd60b9f 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -12,7 +12,10 @@ import java.util.Optional; import java.util.PriorityQueue; import java.util.Set; +import org.apache.commons.collections4.map.HashedMap; + import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; @@ -72,8 +75,8 @@ public class TextEntity implements IEntity { List intersectingNodes = new LinkedList<>(); SemanticNode deepestFullyContainingNode; -// @Builder.Default -// Set relations = new HashSet<>(); + @Builder.Default + HashedMap relations = new HashedMap<>(); public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) { @@ -165,13 +168,15 @@ public class TextEntity implements IEntity { public void removeFromGraph() { + remove("FINAL.0.0", "removed completely"); intersectingNodes.forEach(node -> node.getEntities().remove(this)); pages.forEach(page -> page.getEntities().remove(this)); intersectingNodes = new LinkedList<>(); - //relations = new HashSet<>(); + relations.keySet() + .forEach(entity -> entity.relations.remove(this)); + relations = new HashedMap<>(); deepestFullyContainingNode = null; pages = new HashSet<>(); - remove("FINAL.0.0", "removed completely"); } @@ -259,6 +264,20 @@ public class TextEntity implements IEntity { } + public void addManualChange(BaseAnnotation manualChange) { + + manualOverwrite.addChange(manualChange); + update(); + } + + + public void addManualChanges(List manualChanges) { + + manualOverwrite.addChanges(manualChanges); + update(); + } + + public boolean matchesAnnotationId(String manualRedactionId) { return getPositionsOnPagePerPage().stream() @@ -325,6 +344,7 @@ public class TextEntity implements IEntity { getKieSessionUpdater().ifPresent(updater -> updater.update(this)); } + public void remove() { getKieSessionUpdater().ifPresent(updater -> updater.remove(this)); diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityCreationUtility.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityCreationUtility.java new file mode 100644 index 00000000..2e0afd66 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityCreationUtility.java @@ -0,0 +1,95 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.util.List; +import java.util.Set; + +import com.iqser.red.service.redaction.v1.server.model.document.IntersectingNodeVisitor; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class EntityCreationUtility { + + public void checkIfBothStartAndEndAreEmpty(String start, String end) { + + checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end)); + } + + + public void checkIfBothStartAndEndAreEmpty(List start, List end) { + + if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) { + throw new IllegalArgumentException("Start and end values are empty!"); + } + } + + + public int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) { + + if (textBlock.getNextLinebreak(end) < expandedEnd) { + return end; + } + return expandedEnd; + } + + + public Set findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) { + + IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange); + + if (initialIntersectingNode.getTextRange().intersects(textRange)) { + initialIntersectingNode.accept(visitor); + } + + return visitor.getIntersectingNodes(); + } + + + public void addToPages(TextEntity entity) { + + Set pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange()); + entity.getPages().addAll(pages); + pages.forEach(page -> page.getEntities().add(entity)); + } + + + public void addEntityToNodeEntitySets(TextEntity entity) { + + entity.getIntersectingNodes() + .forEach(node -> node.getEntities().add(entity)); + } + + + public boolean allEntitiesIntersectAndHaveSameTypes(List entitiesToMerge) { + + if (entitiesToMerge.isEmpty()) { + return true; + } + TextEntity previousEntity = entitiesToMerge.get(0); + for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) { + boolean typeMatches = textEntity.type().equals(previousEntity.type()); + boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType()); + boolean intersects = textEntity.intersects(previousEntity); + if (!typeMatches || !entityTypeMatches || !intersects) { + return false; + } + } + return true; + } + + + public TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) { + + if (textBlock.getTextRange().end() == textRange.end()) { + return new TextRange(textRange.end(), textRange.end()); + } + + return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityEnrichmentService.java similarity index 68% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java rename to redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityEnrichmentService.java index 4cc70ca9..fbdd30f5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityEnrichmentService.java @@ -1,39 +1,36 @@ -package com.iqser.red.service.redaction.v1.server.service.document; +package com.iqser.red.service.redaction.v1.server.utils; import java.util.Arrays; import java.util.List; import java.util.Objects; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; -import lombok.RequiredArgsConstructor; +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; +import lombok.experimental.UtilityClass; -@Service -@RequiredArgsConstructor +@UtilityClass +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class EntityEnrichmentService { - private final RedactionServiceSettings redactionServiceSettings; + int SURROUNDING_WORDS_OFFSET_WINDOW = 100; + int NUMBER_OF_SURROUNDING_WORDS = 3; public void enrichEntity(TextEntity entity, TextBlock textBlock) { - entity.setValue(textBlock.subSequence(entity.getTextRange()).toString()); entity.setTextAfter(findTextAfter(entity.getTextRange().end(), textBlock)); entity.setTextBefore(findTextBefore(entity.getTextRange().start(), textBlock)); } - private String findTextAfter(int index, TextBlock textBlock) { - - int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().end()); + int endOffset = Math.min(index + SURROUNDING_WORDS_OFFSET_WINDOW, textBlock.getTextRange().end()); String textAfter = textBlock.subSequence(index, endOffset).toString(); if (!textAfter.isBlank()) { List wordsAfter = splitToWordsAndRemoveEmptyWords(textAfter); - int numberOfWordsAfter = Math.min(wordsAfter.size(), redactionServiceSettings.getNumberOfSurroundingWords()); + int numberOfWordsAfter = Math.min(wordsAfter.size(), NUMBER_OF_SURROUNDING_WORDS); if (!wordsAfter.isEmpty()) { return concatWordsAfter(wordsAfter.subList(0, numberOfWordsAfter), textAfter.startsWith(" ")); } @@ -41,14 +38,12 @@ public class EntityEnrichmentService { return ""; } - private String findTextBefore(int index, TextBlock textBlock) { - - int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().start()); + int offsetBefore = Math.max(index - SURROUNDING_WORDS_OFFSET_WINDOW, textBlock.getTextRange().start()); String textBefore = textBlock.subSequence(offsetBefore, index).toString(); if (!textBefore.isBlank()) { List wordsBefore = splitToWordsAndRemoveEmptyWords(textBefore); - int numberOfWordsBefore = Math.min(wordsBefore.size(), redactionServiceSettings.getNumberOfSurroundingWords()); + int numberOfWordsBefore = Math.min(wordsBefore.size(), NUMBER_OF_SURROUNDING_WORDS); if (!wordsBefore.isEmpty()) { return concatWordsBefore(wordsBefore.subList(wordsBefore.size() - numberOfWordsBefore, wordsBefore.size()), textBefore.endsWith(" ")); } @@ -56,36 +51,26 @@ public class EntityEnrichmentService { return ""; } - - private static List splitToWordsAndRemoveEmptyWords(String textAfter) { - - return Arrays.stream(textAfter.split(" ")) + private List splitToWordsAndRemoveEmptyWords(String text) { + return Arrays.stream(text.split(" ")) .filter(word -> !Objects.equals("", word)) .toList(); } - - private static String concatWordsBefore(List words, boolean endWithSpace) { - + private String concatWordsBefore(List words, boolean endWithSpace) { StringBuilder sb = new StringBuilder(); - for (String word : words) { sb.append(word).append(" "); } - String result = sb.toString().trim(); return endWithSpace ? result + " " : result; } - - private static String concatWordsAfter(List words, boolean startWithSpace) { - + private String concatWordsAfter(List words, boolean startWithSpace) { StringBuilder sb = new StringBuilder(); - for (String word : words) { sb.append(word).append(" "); } - String result = sb.toString().trim(); return startWithSpace ? " " + result : result; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index 1e4f040e..fee7bad2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -11,6 +11,7 @@ import org.apache.commons.lang3.StringUtils; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.utils.Patterns; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; @@ -44,6 +45,13 @@ public class Dictionary { } + public boolean containsType(String type) { + + Map levelMap = localAccessMap.get(type); + return !(levelMap == null || levelMap.isEmpty()); + } + + private Level getLevel(boolean isDossierDictionary) { return isDossierDictionary ? Level.DOSSIER : Level.DOSSIER_TEMPLATE; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 4cd49974..29139505 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -8,10 +8,8 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; @@ -25,9 +23,6 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DictionarySearchService { - EntityEnrichmentService entityEnrichmentService; - RelationStore relationStore; - @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, List semanticNodes) { @@ -39,7 +34,7 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + EntityCreationService entityCreationService = new EntityCreationService(); dictionary.getDictionarySearch().getBoundaries(node.getTextBlock()) .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary.textRange())) .forEach(match -> { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java index c06c4838..31174f1b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java @@ -14,7 +14,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.google.common.collect.Sets; @@ -24,14 +23,12 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplemen import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.RectangleWithPage; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import lombok.extern.slf4j.Slf4j; @@ -40,15 +37,7 @@ import lombok.extern.slf4j.Slf4j; @Service public class EntityFindingUtility { - EntityCreationService entityCreationService; - - - @Autowired - public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService, RelationStore relationStore) { - - entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); - } - + EntityCreationService entityCreationService = new EntityCreationService(); public Optional findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map> entitiesWithSameValue, @@ -192,7 +181,7 @@ public class EntityFindingUtility { return textBlocks.stream() .flatMap(searchImplementation::getBoundaries) - .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet())) + .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, node, Collections.emptySet())) .filter(Optional::isPresent) .map(Optional::get) .distinct() @@ -219,7 +208,7 @@ public class EntityFindingUtility { return textBlocks.stream() .flatMap(tb -> searchImplementation.getBoundaries(tb) .filter(textRange -> entityCreationService.isValidEntityTextRange(tb, textRange))) - .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, document, Collections.emptySet())) + .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, document, Collections.emptySet())) .filter(Optional::isPresent) .map(Optional::get) .distinct() @@ -233,7 +222,7 @@ public class EntityFindingUtility { return searchImplementation.getBoundaries(document.getTextBlock()) .filter(textRange -> entityCreationService.isValidEntityTextRange(document.getTextBlock(), textRange)) - .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, document, Collections.emptySet())) + .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, document, Collections.emptySet())) .filter(Optional::isPresent) .map(Optional::get) .distinct() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index ba644071..c92d0d42 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -431,12 +431,11 @@ public class EntityLogCreatorService { private static EntryType getEntryType(EntityType entityType) { return switch (entityType) { - case ENTITY -> EntryType.ENTITY; + case ENTITY, TEMPORARY -> EntryType.ENTITY; case HINT -> EntryType.HINT; - case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; + case FALSE_POSITIVE, DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE; case RECOMMENDATION -> EntryType.RECOMMENDATION; case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; - case DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE; }; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index 42f10e75..dd1bde9d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -17,6 +17,8 @@ import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; @@ -44,18 +46,19 @@ public class ManualChangesApplicationService { public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) { entityToBeReCategorized.getMatchedRuleList().clear(); - entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization); if (manualRecategorization.getType() == null) { return; } if (entityToBeReCategorized instanceof Image image) { + entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization); image.setImageType(ImageType.fromString(manualRecategorization.getType())); return; } if (entityToBeReCategorized instanceof TextEntity textEntity) { + textEntity.addManualChange(manualRecategorization); textEntity.setType(manualRecategorization.getType()); } } @@ -87,7 +90,7 @@ public class ManualChangesApplicationService { .map(ManualChangesApplicationService::toRectangle2D) .collect(Collectors.toList())); - entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction); + entityToBeResized.addManualChange(manualResizeRedaction); SemanticNode node = entityToBeResized.getDeepestFullyContainingNode(); PrecursorEntity searchEntity = PrecursorEntity.fromManualResizeRedaction(manualResizeRedaction); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index 4b19b877..ba5d7216 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -1,17 +1,10 @@ package com.iqser.red.service.redaction.v1.server.service.document; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addEntityToNodeEntitySets; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addToPages; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.allEntitiesIntersectAndHaveSameTypes; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.checkIfBothStartAndEndAreEmpty; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.findIntersectingSubNodes; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.toLineAfterTextRange; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween; +import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*; import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; import java.util.List; @@ -29,44 +22,26 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplemen import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality; import com.iqser.red.service.redaction.v1.server.model.document.entity.IKieSessionUpdater; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@RequiredArgsConstructor public class EntityCreationService { - private final EntityEnrichmentService entityEnrichmentService; - private final RelationStore relationStore; - private final Set nodesInKieSession; // empty set means all nodes are in kieSession - - - public EntityCreationService(EntityEnrichmentService entityEnrichmentService, RelationStore relationStore) { - - this.entityEnrichmentService = entityEnrichmentService; - this.relationStore = relationStore; - this.nodesInKieSession = Collections.emptySet(); - } - /** * Creates entities found between specified start and stop strings, case-sensitive. @@ -1010,12 +985,14 @@ public class EntityCreationService { } return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there } - boolean added = addEntityToGraph(entity, node.getDocumentTree()); + boolean added = node.getDocumentTree().addEntityToGraph(entity); if (!added) { return Optional.empty(); } entity.addEngines(engines); - insertToKieSession(entity, node); + if(!entityType.equals(EntityType.TEMPORARY)) { + insertToKieSession(entity, node); + } return Optional.of(entity); } @@ -1080,14 +1057,14 @@ public class EntityCreationService { .map(TextEntity::getManualOverwrite) .map(ManualChangeOverwrite::getManualChangeLog) .flatMap(Collection::stream) - .forEach(manualChange -> mergedEntity.getManualOverwrite().addChange(manualChange)); + .forEach(mergedEntity::addManualChange); mergedEntity.setDictionaryEntry(entitiesToMerge.stream() .anyMatch(TextEntity::isDictionaryEntry)); mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream() .anyMatch(TextEntity::isDossierDictionaryEntry)); - entityEnrichmentService.enrichEntity(mergedEntity, node.getTextBlock()); + EntityEnrichmentService.enrichEntity(mergedEntity, node.getTextBlock()); addEntityToGraph(mergedEntity, node); insertToKieSession(mergedEntity, node); @@ -1145,7 +1122,7 @@ public class EntityCreationService { TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException( "No entity present!")); - newEntity.getManualOverwrite().addChanges(entity.getManualOverwrite().getManualChangeLog()); + newEntity.addManualChanges(entity.getManualOverwrite().getManualChangeLog()); newEntity.setDictionaryEntry(entity.isDictionaryEntry()); newEntity.setDossierDictionaryEntry(entity.isDossierDictionaryEntry()); return newEntity; @@ -1456,7 +1433,7 @@ public class EntityCreationService { .ifPresent(e -> addDuplicateEntityToGraph(e, entity.getTextRange(), node)); } else { - addEntityToGraph(entity, documentTree); + documentTree.addEntityToGraph(entity); } } @@ -1507,48 +1484,4 @@ public class EntityCreationService { }); } - - private boolean addEntityToGraph(TextEntity entity, DocumentTree documentTree) { - - documentTree.getRoot().getNode().addThisToEntityIfIntersects(entity); - -// if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes() -// .stream() -// .anyMatch(node -> !nodesInKieSession.contains(node))) { -// entity.removeFromGraph(); -// return false; -// } - - TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock(); - entityEnrichmentService.enrichEntity(entity, textBlock); - - addToPages(entity); - addEntityToNodeEntitySets(entity); - - if(relationStore != null) { - //List relations = new LinkedList<>(); - // compute relationships, by looping through deepestFullyContainingNode's entities - for (TextEntity textEntity : entity.getDeepestFullyContainingNode().getEntities()) { - if (entity.intersects(textEntity) && !entity.equals(textEntity)) { - Relation relation; - if (textEntity.getTextRange().equals(entity.getTextRange())) { - relation = new Equality(entity, textEntity); - } else if (textEntity.containedBy(entity)) { - relation = new Containment(textEntity, entity); - } else if (entity.containedBy(textEntity)) { - relation = new Containment(entity, textEntity); - } else { - relation = new Intersection(entity, textEntity); - } - relationStore.addRelation(relation); - //relations.add(relation); - //textEntity.getRelations().add(relation); - } - } - //entity.getRelations().addAll(relations); - } - - return true; - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java index 59c3fd4c..a6102e4d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java @@ -9,15 +9,18 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import lombok.experimental.UtilityClass; + +@UtilityClass public class EntityCreationUtility { - public static void checkIfBothStartAndEndAreEmpty(String start, String end) { + public void checkIfBothStartAndEndAreEmpty(String start, String end) { checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end)); } - public static void checkIfBothStartAndEndAreEmpty(List start, List end) { + public void checkIfBothStartAndEndAreEmpty(List start, List end) { if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) { throw new IllegalArgumentException("Start and end values are empty!"); @@ -25,7 +28,7 @@ public class EntityCreationUtility { } - public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) { + public int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) { if (textBlock.getNextLinebreak(end) < expandedEnd) { return end; @@ -34,7 +37,7 @@ public class EntityCreationUtility { } - public static Set findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) { + public Set findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) { IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange); @@ -46,7 +49,7 @@ public class EntityCreationUtility { } - public static void addToPages(TextEntity entity) { + public void addToPages(TextEntity entity) { Set pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange()); entity.getPages().addAll(pages); @@ -54,14 +57,14 @@ public class EntityCreationUtility { } - public static void addEntityToNodeEntitySets(TextEntity entity) { + public void addEntityToNodeEntitySets(TextEntity entity) { entity.getIntersectingNodes() .forEach(node -> node.getEntities().add(entity)); } - public static boolean allEntitiesIntersectAndHaveSameTypes(List entitiesToMerge) { + public boolean allEntitiesIntersectAndHaveSameTypes(List entitiesToMerge) { if (entitiesToMerge.isEmpty()) { return true; @@ -79,7 +82,7 @@ public class EntityCreationUtility { } - public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) { + public TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) { if (textBlock.getTextRange().end() == textRange.end()) { return new TextRange(textRange.end(), textRange.end()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index 127d523d..39a08267 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.service.drools; import static com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService.RULES_LOGGER_GLOBAL; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -11,7 +10,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieSession; @@ -30,13 +28,10 @@ import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; import com.iqser.red.service.redaction.v1.server.logger.TrackingAgendaEventListener; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.websocket.WebSocketService; import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; @@ -54,8 +49,6 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class EntityDroolsExecutionService { - EntityEnrichmentService entityEnrichmentService; - RelationStore relationStore; ObservationRegistry observationRegistry; ManualChangesApplicationService manualChangesApplicationService; RedactionServiceSettings settings; @@ -99,113 +92,87 @@ public class EntityDroolsExecutionService { KieSession kieSession = kieContainer.newKieSession(); - Set nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes() - .count() ? new HashSet<>() : buildSet(sectionsToAnalyze, document); - - KieSessionUpdater kieSessionUpdater = new KieSessionUpdater(kieSession, relationStore, nodesInKieSession); - document.setKieSessionUpdater(kieSessionUpdater); - - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore, nodesInKieSession); - RulesLogger logger = new RulesLogger(webSocketService, context); - if (settings.isDroolsDebug()) { - logger.enableAgendaTracking(); - logger.enableObjectTracking(); - } - kieSession.addEventListener(new TrackingAgendaEventListener(logger)); - kieSession.addEventListener(new ObjectTrackingEventListener(logger)); - - kieSession.setGlobal("document", document); - kieSession.setGlobal("entityCreationService", entityCreationService); - kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); - kieSession.setGlobal("dictionary", dictionary); - - if (hasGlobalWithName(kieSession, RULES_LOGGER_GLOBAL)) { - kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger); - } - - nodesInKieSession.add(document); - nodesInKieSession.addAll(sectionsToAnalyze); - nodesInKieSession.addAll(sectionsToAnalyze.stream() - .flatMap(SemanticNode::streamAllSubNodes) - .toList()); - nodesInKieSession.forEach(kieSession::insert); - System.out.println("after document insert : " + kieSession.getFactCount()); - - document.getEntities() - .forEach(kieSession::insert); - relationStore.getAllRelations(document.getEntities() - .stream() - .map(TextEntity::getId) - .collect(Collectors.toSet())) - .forEach(kieSession::insert); - System.out.println("after getEntities insert : " + kieSession.getFactCount()); - - document.getPages() - .forEach(kieSession::insert); - System.out.println("after getPages insert : " + kieSession.getFactCount()); - - fileAttributes.stream() - .filter(f -> f.getValue() != null) - .forEach(kieSession::insert); - System.out.println("after fileAttributes insert : " + kieSession.getFactCount()); - - if (manualRedactions != null) { - manualRedactions.buildAll() - .stream() - .filter(BaseAnnotation::isLocal) - .forEach(kieSession::insert); - } - System.out.println("after manualRedactions insert : " + kieSession.getFactCount()); - - kieSession.insert(nerEntities); - System.out.println("after nerEntities insert : " + kieSession.getFactCount()); - - kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); - - CompletableFuture completableFuture = CompletableFuture.supplyAsync(() -> { - kieSession.fireAllRules(); - return null; - }); - try { - completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS); -// -// nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes() -// .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document); -// -// kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession); -// -// document.setKieSessionUpdater(kieSessionUpdater); -// -// kieSession.insert(document); -// -// kieSession.setGlobal("entityCreationService", entityCreationService); -// -// sectionsToAnalyze.stream() -// .flatMap(SemanticNode::streamAllSubNodes) -// .forEach(kieSession::insert); -// System.out.println("after SemanticNode insert : " + kieSession.getFactCount()); -// -// completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS); + KieSessionUpdater kieSessionUpdater = new KieSessionUpdater(kieSession); + document.setKieSessionUpdater(kieSessionUpdater); - } catch (ExecutionException e) { - logger.error(e, "Exception during rule execution"); - kieSession.dispose(); - if (e.getCause() instanceof TimeoutException) { + EntityCreationService entityCreationService = new EntityCreationService(); + RulesLogger logger = new RulesLogger(webSocketService, context); + if (settings.isDroolsDebug()) { + logger.enableAgendaTracking(); + logger.enableObjectTracking(); + } + kieSession.addEventListener(new TrackingAgendaEventListener(logger)); + kieSession.addEventListener(new ObjectTrackingEventListener(logger)); + + kieSession.setGlobal("document", document); + kieSession.setGlobal("entityCreationService", entityCreationService); + kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); + kieSession.setGlobal("dictionary", dictionary); + + if (hasGlobalWithName(kieSession, RULES_LOGGER_GLOBAL)) { + kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger); + } + + kieSession.insert(document); + sectionsToAnalyze.forEach(kieSession::insert); + sectionsToAnalyze.stream() + .flatMap(SemanticNode::streamAllSubNodes) + .forEach(kieSession::insert); + System.out.println("after document insert : " + kieSession.getFactCount()); + + document.getEntities() + .forEach(kieSession::insert); + document.getEntities() + .forEach(textEntity -> textEntity.getRelations().values() + .forEach(kieSession::insert)); + System.out.println("after getEntities insert : " + kieSession.getFactCount()); + + document.getPages() + .forEach(kieSession::insert); + System.out.println("after getPages insert : " + kieSession.getFactCount()); + + fileAttributes.stream() + .filter(f -> f.getValue() != null) + .forEach(kieSession::insert); + System.out.println("after fileAttributes insert : " + kieSession.getFactCount()); + + if (manualRedactions != null) { + manualRedactions.buildAll() + .stream() + .filter(BaseAnnotation::isLocal) + .forEach(kieSession::insert); + } + System.out.println("after manualRedactions insert : " + kieSession.getFactCount()); + + kieSession.insert(nerEntities); + System.out.println("after nerEntities insert : " + kieSession.getFactCount()); + + kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); + + CompletableFuture completableFuture = CompletableFuture.runAsync(kieSession::fireAllRules); + + try { + completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS); + } catch (ExecutionException e) { + logger.error(e, "Exception during rule execution"); + if (e.getCause() instanceof TimeoutException) { + throw new DroolsTimeoutException(String.format("The file %s caused a timeout", context.getFileId()), e, false, RuleFileType.ENTITY); + } + throw new RuntimeException(e); + } catch (InterruptedException e) { + logger.error(e, "Exception during rule execution"); + throw new RuntimeException(e); + } catch (TimeoutException e) { throw new DroolsTimeoutException(String.format("The file %s caused a timeout", context.getFileId()), e, false, RuleFileType.ENTITY); } - throw new RuntimeException(e); - } catch (InterruptedException e) { - logger.error(e, "Exception during rule execution"); - kieSession.dispose(); - throw new RuntimeException(e); - } catch (TimeoutException e) { - throw new DroolsTimeoutException(String.format("The file %s caused a timeout", context.getFileId()), e, false, RuleFileType.ENTITY); - } - List resultingFileAttributes = getFileAttributes(kieSession); - kieSession.dispose(); - return resultingFileAttributes; + List resultingFileAttributes = getFileAttributes(kieSession); + return resultingFileAttributes; + + } finally { + kieSession.dispose(); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java index 0503bcd6..a1cfc267 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java @@ -1,13 +1,9 @@ package com.iqser.red.service.redaction.v1.server.service.drools; -import java.util.Set; - import org.kie.api.runtime.KieSession; import org.kie.api.runtime.rule.FactHandle; import com.iqser.red.service.redaction.v1.server.model.document.entity.IKieSessionUpdater; -import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; @@ -21,16 +17,17 @@ import lombok.experimental.FieldDefaults; public class KieSessionUpdater implements IKieSessionUpdater { KieSession kieSession; - RelationStore relationStore; - Set nodesInKieSession; public void insert(TextEntity textEntity) { kieSession.insert(textEntity); updateIntersectingNodes(textEntity); - relationStore.getRelations(textEntity.getId()) + textEntity.getRelations().values() .forEach(kieSession::insert); + textEntity.getRelations().keySet() + .forEach(k -> kieSession.insert(k.getRelations() + .get(textEntity))); } @@ -38,11 +35,15 @@ public class KieSessionUpdater implements IKieSessionUpdater { kieSession.update(kieSession.getFactHandle(textEntity), textEntity); updateIntersectingNodes(textEntity); - relationStore.getRelations(textEntity.getId()) - .forEach(r -> kieSession.update(kieSession.getFactHandle(r), r)); + textEntity.getRelations().values() + .forEach(this::updateFactIfPresent); + textEntity.getRelations().keySet() + .forEach(k -> updateFactIfPresent(k.getRelations() + .get(textEntity))); } + public void update(Image image) { kieSession.update(kieSession.getFactHandle(image), image); @@ -56,15 +57,14 @@ public class KieSessionUpdater implements IKieSessionUpdater { public void remove(TextEntity textEntity) { + //test replace all deletes with updates kieSession.delete(kieSession.getFactHandle(textEntity)); updateIntersectingNodes(textEntity); - relationStore.getRelations(textEntity.getId()) - .forEach(r -> { - FactHandle factHandle = kieSession.getFactHandle(r); - if (factHandle != null) { - kieSession.delete(factHandle); - } - }); + textEntity.getRelations().values() + .forEach(this::deleteFactIfPresent); + textEntity.getRelations().keySet() + .forEach(k -> deleteFactIfPresent(k.getRelations() + .get(textEntity))); } @@ -82,9 +82,25 @@ public class KieSessionUpdater implements IKieSessionUpdater { private void updateIntersectingNodes(TextEntity textEntity) { textEntity.getIntersectingNodes() - .stream() - .filter(nodesInKieSession::contains) - .forEach(o -> kieSession.update(kieSession.getFactHandle(o), o)); + .forEach(this::updateFactIfPresent); + } + + + + private void updateFactIfPresent(Object o) { + + FactHandle factHandle = kieSession.getFactHandle(o); + if (factHandle != null) { + kieSession.update(factHandle, o); + } + } + + private void deleteFactIfPresent(Object o) { + + FactHandle factHandle = kieSession.getFactHandle(o); + if (factHandle != null) { + kieSession.delete(factHandle); + } } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl index 7e26fafb..3be0cf9c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl @@ -17,8 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; @@ -1334,7 +1333,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -1358,7 +1357,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -1383,7 +1382,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1394,7 +1393,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1438,7 +1437,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl index 489bad7b..e8152dd3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl @@ -18,8 +18,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java index aba510f7..5f1fed95 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java @@ -7,6 +7,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.wildfly.common.Assert.assertTrue; +import java.util.HashSet; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -14,12 +15,10 @@ import org.junit.jupiter.api.Test; import org.kie.api.runtime.KieSession; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; @@ -31,14 +30,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.drools.KieSessionUpdater; public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; - @Autowired - private RelationStore relationStore; private EntityCreationService entityCreationService; @Mock @@ -49,7 +44,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr public void createEntityCreationService() { MockitoAnnotations.initMocks(this); - entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + entityCreationService = new EntityCreationService(); } @@ -93,6 +88,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr public void assertSameEntitiesCantBeCreatedTwice() { Document document = buildGraph("files/new/crafted document.pdf"); + document.setKieSessionUpdater(new KieSessionUpdater(kieSession)); String type = "CBI_author"; assertTrue(entityCreationService.byTextRange(new TextRange(0, 10), type, EntityType.ENTITY, document) .isPresent()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index 3e6fb4a6..78442fb6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -30,7 +30,6 @@ import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; @@ -40,7 +39,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -54,10 +52,6 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest { @Autowired private DictionaryService dictionaryService; - @Autowired - private EntityEnrichmentService entityEnrichmentService; - @Autowired - private RelationStore relationStore; private EntityCreationService entityCreationService; @Autowired @@ -71,7 +65,7 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest { @BeforeEach public void stubClients() { - entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + entityCreationService = new EntityCreationService(); TenantContext.setTenantId("redaction"); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java index 8abb56d3..43e7f9f1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java @@ -5,32 +5,22 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.List; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; public class SearchImplementationTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; - - @Autowired - private RelationStore relationStore; - - @Test public void testSearchImplementationWithPunctuation() { Document document = buildGraph("files/Minimal Examples/TestPunctuation"); SearchImplementation searchImplementation = new SearchImplementation(List.of("Kuhn, J. O."), true); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + EntityCreationService entityCreationService = new EntityCreationService(); List entities = entityCreationService.bySearchImplementation(searchImplementation, "CBI_author", EntityType.ENTITY, document) .toList(); assertEquals(2, entities.size()); @@ -42,7 +32,7 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest { Document document = buildGraph("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections"); SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + EntityCreationService entityCreationService = new EntityCreationService(); List entities = entityCreationService.bySearchImplementation(searchImplementation, "dossier_redaction", EntityType.ENTITY, document) .toList(); assertEquals(2, entities.size()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java index 1c89feb3..c2e76757 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java @@ -12,17 +12,14 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility; import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService; @@ -33,12 +30,6 @@ import lombok.SneakyThrows; public class TableTest extends BuildDocumentIntegrationTest { private static final boolean DRAW_FILE = false; - - @Autowired - private EntityEnrichmentService entityEnrichmentService; - @Autowired - private RelationStore relationStore; - private EntityCreationService entityCreationService; private static final String TYPE_1 = "type1"; @@ -55,7 +46,7 @@ public class TableTest extends BuildDocumentIntegrationTest { @BeforeEach public void createTable() { - entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + entityCreationService = new EntityCreationService(); String fileName = "files/Minimal Examples/BasicTable.pdf"; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index b8fd95fe..79f514f3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -23,7 +23,6 @@ import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; @@ -36,8 +35,6 @@ class DroolsValidationServiceTest { @MockBean RulesClient rulesClient; @MockBean - EntityEnrichmentService entityEnrichmentService; - @MockBean RedactionServiceSettings redactionServiceSettings; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 0b486730..bc549df1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -23,7 +23,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -51,11 +50,9 @@ import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -69,11 +66,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl"); private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl"); - @Autowired - private EntityEnrichmentService entityEnrichmentService; - - @Autowired - private RelationStore relationStore; private EntityCreationService entityCreationService; @@ -81,7 +73,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { @BeforeEach public void createServices() { - entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + entityCreationService = new EntityCreationService(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index ce91c661..d0097345 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -15,16 +15,22 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import com.google.common.collect.Sets; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.redaction.v1.server.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest; +import com.iqser.red.service.redaction.v1.server.service.drools.KieSessionUpdater; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; + +import lombok.SneakyThrows; public class ManualChangesIntegrationTest extends RulesIntegrationTest { @@ -225,4 +231,14 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest { return new Rectangle((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number); } + + @Override + @SneakyThrows + protected Document buildGraph(String filename) { + + Document document = super.buildGraph(filename); + document.setKieSessionUpdater(new KieSessionUpdater(kieSession)); + return document; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index 07ba02bd..b5160d8a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -17,19 +17,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; - - @Autowired - private RelationStore relationStore; private EntityCreationService entityCreationService; @@ -37,7 +30,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { @BeforeEach public void createServices() { - entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + entityCreationService = new EntityCreationService(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java index 32c1e701..8f7deae9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java @@ -30,24 +30,17 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService; import lombok.SneakyThrows; public class PrecursorEntityTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; - - @Autowired - private RelationStore relationStore; @Autowired private EntityFromPrecursorCreationService entityFromPrecursorCreationService; @@ -108,7 +101,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { public void createFoundManualRedaction2() { Document document = buildGraph("files/Minimal Examples/TestPunctuation"); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + EntityCreationService entityCreationService = new EntityCreationService(); List tempEntities = entityCreationService.byString("Kuhn, J. O.", "CBI_author", EntityType.ENTITY, document) .toList(); @@ -163,7 +156,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { private DocumentAndEntity createFoundManualRedaction() { Document document = buildGraph("files/syngenta/CustomerFiles/VV-919901.pdf"); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + EntityCreationService entityCreationService = new EntityCreationService(); List tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document) .toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index 8ff6d22a..e3592853 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -19,7 +19,6 @@ import org.apache.pdfbox.Loader; import org.apache.pdfbox.pdmodel.PDDocument; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import com.iqser.red.commons.jackson.ObjectMapperFactory; @@ -29,11 +28,9 @@ import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter; import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; @@ -43,10 +40,6 @@ import lombok.extern.slf4j.Slf4j; @Slf4j class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; - @Autowired - private RelationStore relationStore; private EntityCreationService entityCreationService; @@ -61,7 +54,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { @BeforeEach public void createEntityCreationService() { - entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + entityCreationService = new EntityCreationService(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java index 7276acb2..a470a436 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java @@ -22,11 +22,9 @@ import org.springframework.context.annotation.Import; import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; @ExtendWith(MockitoExtension.class) @Import(RulesIntegrationTest.TestConfiguration.class) @@ -34,10 +32,6 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { protected static final String RULES = "drools/rules.drl"; - @Autowired - protected EntityEnrichmentService entityEnrichmentService; - @Autowired - protected RelationStore relationStore; @Autowired protected ManualChangesApplicationService manualChangesApplicationService; protected EntityCreationService entityCreationService; @@ -85,7 +79,7 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { Dictionary dict = Mockito.mock(Dictionary.class); kieSession = kieContainer.newKieSession(); - entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); + entityCreationService = new EntityCreationService(); kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); kieSession.setGlobal("entityCreationService", entityCreationService); kieSession.setGlobal("dictionary", dict); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt index 54a53703..1c68c41a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt @@ -1660,104 +1660,4 @@ Zyma SA Zyma SA, Nyon, Switzerland Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK -Test Ignored Hint CBI_ADDRESS -the -be -to -of -and -a -in -that -have -I -it -for -not -on -with -he -as -you -do -at -this -but -his -by -from -they -we -say -her -she -or -an -will -my -one -all -would -there -their -what -so -up -out -if -about -who -get -which -go -me -when -make -can -like -time -no -just -him -know -take -people -into -year -your -good -some -could -them -see -other -than -then -now -look -only -come -its -over -think -also -back -after -use -two -how -our -work -first -well -way -even -new -want -because -any -these -give -day -most -us \ No newline at end of file +Test Ignored Hint CBI_ADDRESS \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index d5535a2a..f85f8695 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -1135,7 +1134,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -1159,7 +1158,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -1184,7 +1183,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1195,7 +1194,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1239,7 +1238,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 81897adc..d9d3b84b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -1899,7 +1898,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -1923,7 +1922,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -1948,7 +1947,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1959,7 +1958,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -2003,7 +2002,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 29ae8032..a75394da 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -1271,7 +1270,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -1295,7 +1294,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -1320,7 +1319,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1331,7 +1330,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1375,7 +1374,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index cacfe1e1..9e6a4bd8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -862,7 +861,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -886,7 +885,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -911,7 +910,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -922,7 +921,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -966,7 +965,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 09157bdc..ab35fa3f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -183,7 +182,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -207,7 +206,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -232,7 +231,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -243,7 +242,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -287,7 +286,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 57af3cd3..065fa322 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -18,11 +18,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment -import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality -import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -1158,7 +1157,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -1182,7 +1181,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -1207,7 +1206,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1218,7 +1217,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1262,7 +1261,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -1499,21 +1498,6 @@ rule "X.10.0: remove false positives of ai" end // Rule unit: X.11 -rule "X.11.0: Remove dictionary entity which intersects with a manual entity" - salience 64 - when - $intersection: Intersection( - $a: a, - $b: b, - $a.engines contains Engine.MANUAL, - $a.active(), - $b.dictionaryEntry, - $b.engines not contains Engine.MANUAL - ) - then - $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - end - rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 7ebdfa5a..b7f2d0ea 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -254,7 +253,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -278,7 +277,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -303,7 +302,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -314,7 +313,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -345,7 +344,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index a4cce953..dbc53316 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -333,7 +332,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -357,7 +356,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -382,7 +381,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -393,7 +392,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -437,7 +436,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index dda0ce3f..cda6d6c0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -233,7 +232,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -257,7 +256,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -282,7 +281,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -293,7 +292,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -337,7 +336,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index e9fc33c6..d1e0bc07 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -746,7 +745,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -770,7 +769,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -795,7 +794,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -806,7 +805,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -850,7 +849,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index f7954135..37dcfb76 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -1917,7 +1916,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -1941,7 +1940,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -1966,7 +1965,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1977,7 +1976,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -2022,7 +2021,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index bb8475de..15f444c4 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -1338,7 +1337,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); end @@ -1362,7 +1361,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); + $entityToForce.addManualChange($force); retract($force); end @@ -1387,7 +1386,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1398,7 +1397,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1441,7 +1440,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt index 2020709c..3016b7c0 100644 --- a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt +++ b/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt @@ -1 +1 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//---------------------------------------------------------------------------\n\n\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.0.1: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n\n then\n $tocHeadline.getParent().getPages()\n .forEach(page -> page.getMainBody().stream()\n .filter(node -> !node.getType().equals(NodeType.IMAGE))\n .filter(node -> node.getPages().stream().noneMatch(nodePage -> nodePage.getNumber() < page.getNumber()))\n .forEach(node -> retract(node))\n );\n end\n \n\n\n/*\nrule \"H.0.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.RECOMMENDATION);\n end\n*/\n\nrule \"DOC.0.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"408\",\"414\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n onPage(1)\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline, Method No. (\\\\d{3})\", 1, $section.getTextBlock()) \n ).flatMap(Collection::stream).findFirst()\n .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n onPage(1)\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// Examples found in PoC 1\n// entityCreationService.byRegex(\"((OECD Guidelines for Testing of Chemicals, Procedure)|(OECD Guidelines for the Testing of Chemicals No\\\\.)|(OECD Test Guideline)|(OECD \\\\[Test Guideline, Number)) \\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n// entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n// );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n// new approach OECD Guideline\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// missing OECD guideline rules for RFP demo file\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.1: Guidelines\"\n when\n $headline: Headline(\n onPage(1),\n containsString(\"OECD\")\n )\n then\n entityCreationService.byRegex(\"(OECD (No\\\\.? )?(\\\\d{3})( \\\\(\\\\d{4}\\\\))?)\", \"oecd_guideline\", EntityType.ENTITY,1, $headline).forEach(entity ->\n entity.apply(\"DOC.1.1\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\");\n });\n end\n\n\nrule \"DOC.1.3: Remove guidelines from irrelevant sections\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.remove(\"DOC.1.3\", \"remove guidelines from irrelevant sections\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.3.2: Experimental Completion Date\"\n salience 10\n when \n $section: Section(onPage(1) && (containsString(\"STUDY COMPLETED ON\") || containsString(\"STUDY COMPLETION DATE\") || containsString(\"Report completion date\") || containsString(\"Date of Report\") || containsString(\"AMENDMENT COMPLETION DATE\") || containsString(\"AMENDMENT COMPLETED ON\")))\n \n then\n entityCreationService.byRegex(\"STUDY COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"STUDY COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Report completion date (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Date of Report (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n end\n\n\n\n\n\n // hide all skipped species and strains except in the relevant sections\n rule \"DOC.4.2: Species\"\n salience 1\n when\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.remove(\"DOC.4.2\",\"n-a\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.4.3: Species\"\n when\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.4.3\", \"Species found.\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.0: Strain\"\n when\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.0\", \"Strain found.\");\n });\n end\n\n\n\n\nrule \"DOC.35.0: Sex\"\n when\n \n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.35.0\", \"Test animal sex found\");\n });\n end\n\n\n\nrule \"DOC.6.0: Authors\"\n when\n $headline: Headline(onPage(1), containsString(\"AUTHOR\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"author\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.6.0\", \"Author found\"));\n end\n\n\n\nrule \"DOC.6.2: Authors\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"AUTHOR(S)\" || contains \"AUTHORS\" || contains \"Author\"), getMainBodyTextBlock().getSearchText() (contains \"STUDY COMPLETED ON\" || contains \"STUDY COMPLETION DATE\" || contains \"DATE OF INTERIM REPORT\" || contains \"Report completion date\" || contains \"Date of Report\" || contains \"AMENDMENT COMPLETION DATE\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"AUTHOR(S)\", \"AUTHORS\", \"Author\"),\n List.of(\"STUDY COMPLETED ON\", \"STUDY COMPLETION DATE\", \"DATE OF INTERIM REPORT\", \"Report completion date\", \"Date of Report\", \"AMENDMENT COMPLETION DATE\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.2\", \"Author found\"));\n end\n\n\n\n\nrule \"DOC.6.6: laboratory_project_identification\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Laboratory Project Identification\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\"),\n List.of(\"SPONSOR\", \"VOLUME\", \"This\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.6\", \"Laboratory Project Identification\"));\n end\n\n\n\nrule \"DOC.7.2: study title by document structure\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"STUDY TITLE\" || contains \"Study Title\" || contains \"STUDYTITLE\" || contains \"Report (Final)\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"STUDY TITLE\", \"STUDYTITLE\", \"Report (Final)\"),\n List.of(\"TEST GUIDELINES\", \"TEST GUIDELINE(S)\", \"Guidelines\", \"DATA REQUIREMENT\", \"AUTHOR(S)\", \"AUTHOR\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.7.2\", \"Study title found\"));\n end\n\n\n\nrule \"DOC.8.1: Performing Laboratory\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"PERFORMING LABORATORY\" || contains \"TEST FACILITIES\" || contains \"TEST FACILITY\" || contains \"Test Facility\"), getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Sponsor\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"PERFORMING LABORATORY\", \"TEST FACILITIES\", \"TEST FACILITY\"),\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\", \"Sponsor\", \"PROJECT IDENTIFICATION\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.8.1\", \"Performing Laboratory found\"));\n end\n\n\n\n\nrule \"DOC.8.2: Summary Methods\"\n when\n $headline: Headline(containsString(\"1.1. METHODS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_methods\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.2\", \"Summary Methods found\"));\n end\n\nrule \"DOC.8.3: Summary Observations Laboratory\"\n when\n $headline: Headline(containsString(\"1.2. OBSERVATIONS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_observations\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.3\", \"Summary Observations found\"));\n end\n\n\nrule \"DOC.8.5: Summary Results\"\n when\n Headline((containsStringIgnoreCase(\"1.3. RESULTS\") || containsStringIgnoreCase(\"1.2. RESULTS\")), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> {\n entity.apply(\"DOC.8.5\", \"Summary Results\");\n });\n end\n\nrule \"DOC.8.6: Summary Results 2\"\n when\n $headline: Headline(containsString(\"1.2. RESULTS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\")) \n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.6\", \"Summary Results\"));\n end\n\n\n\nrule \"DOC.8.4: Summary Conclusion\"\n when\n $headline: Headline(containsString(\"1.4. CONCLUSION\") || containsString(\"1.3. CONCLUSION\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_conclusion\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.4\", \"Summary Conculsion found\"));\n end\n\n\n\nrule \"DOC.9.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\")\n || containsString(\"GLP CERTIFICATE (FROM THE COMPETENT AUTHORITY)\")\n || containsString(\"GLP COMPLIANCE STATEMENT\")\n || containsString(\"GLP STATEMENT\") \n )\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.9.0\", \"GLP Study found\");\n });\n end\n\n\n\nrule \"DOC.9.1: GLP Study\"\n when\n $paragraph: Paragraph(containsString(\"GLP COMPLIANCE STATEMENT\"))\n then\n entityCreationService.byRegex(\"GLP COMPLIANCE STATEMENT\", \"glp_study\", EntityType.ENTITY, $paragraph).forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"GLP Study found\");\n });\n end\n\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.getManualOverwrite().addChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: TextEntity($type: type, $entityType: entityType, !resized(), active())\n $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())\n then\n TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);\n $first.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n $second.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n retract($first);\n retract($second);\n mergedEntity.getIntersectingNodes().forEach(node -> update(node));\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())\n then\n $lowerRank.getIntersectingNodes().forEach(node -> update(node));\n $lowerRank.remove(\"X.6.0\", \"remove Entity of lower rank, when intersected by entity of type ENTITY\");\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file +"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//---------------------------------------------------------------------------\n\n\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.0.1: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n\n then\n $tocHeadline.getParent().getPages()\n .forEach(page -> page.getMainBody().stream()\n .filter(node -> !node.getType().equals(NodeType.IMAGE))\n .filter(node -> node.getPages().stream().noneMatch(nodePage -> nodePage.getNumber() < page.getNumber()))\n .forEach(node -> retract(node))\n );\n end\n \n\n\n/*\nrule \"H.0.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.RECOMMENDATION);\n end\n*/\n\nrule \"DOC.0.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"408\",\"414\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n onPage(1)\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline, Method No. (\\\\d{3})\", 1, $section.getTextBlock()) \n ).flatMap(Collection::stream).findFirst()\n .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n onPage(1)\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// Examples found in PoC 1\n// entityCreationService.byRegex(\"((OECD Guidelines for Testing of Chemicals, Procedure)|(OECD Guidelines for the Testing of Chemicals No\\\\.)|(OECD Test Guideline)|(OECD \\\\[Test Guideline, Number)) \\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n// entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n// );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n// new approach OECD Guideline\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// missing OECD guideline rules for RFP demo file\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.1: Guidelines\"\n when\n $headline: Headline(\n onPage(1),\n containsString(\"OECD\")\n )\n then\n entityCreationService.byRegex(\"(OECD (No\\\\.? )?(\\\\d{3})( \\\\(\\\\d{4}\\\\))?)\", \"oecd_guideline\", EntityType.ENTITY,1, $headline).forEach(entity ->\n entity.apply(\"DOC.1.1\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\");\n });\n end\n\n\nrule \"DOC.1.3: Remove guidelines from irrelevant sections\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.remove(\"DOC.1.3\", \"remove guidelines from irrelevant sections\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.3.2: Experimental Completion Date\"\n salience 10\n when \n $section: Section(onPage(1) && (containsString(\"STUDY COMPLETED ON\") || containsString(\"STUDY COMPLETION DATE\") || containsString(\"Report completion date\") || containsString(\"Date of Report\") || containsString(\"AMENDMENT COMPLETION DATE\") || containsString(\"AMENDMENT COMPLETED ON\")))\n \n then\n entityCreationService.byRegex(\"STUDY COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"STUDY COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Report completion date (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Date of Report (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n end\n\n\n\n\n\n // hide all skipped species and strains except in the relevant sections\n rule \"DOC.4.2: Species\"\n salience 1\n when\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.remove(\"DOC.4.2\",\"n-a\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.4.3: Species\"\n when\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.4.3\", \"Species found.\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.0: Strain\"\n when\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.0\", \"Strain found.\");\n });\n end\n\n\n\n\nrule \"DOC.35.0: Sex\"\n when\n \n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.35.0\", \"Test animal sex found\");\n });\n end\n\n\n\nrule \"DOC.6.0: Authors\"\n when\n $headline: Headline(onPage(1), containsString(\"AUTHOR\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"author\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.6.0\", \"Author found\"));\n end\n\n\n\nrule \"DOC.6.2: Authors\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"AUTHOR(S)\" || contains \"AUTHORS\" || contains \"Author\"), getMainBodyTextBlock().getSearchText() (contains \"STUDY COMPLETED ON\" || contains \"STUDY COMPLETION DATE\" || contains \"DATE OF INTERIM REPORT\" || contains \"Report completion date\" || contains \"Date of Report\" || contains \"AMENDMENT COMPLETION DATE\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"AUTHOR(S)\", \"AUTHORS\", \"Author\"),\n List.of(\"STUDY COMPLETED ON\", \"STUDY COMPLETION DATE\", \"DATE OF INTERIM REPORT\", \"Report completion date\", \"Date of Report\", \"AMENDMENT COMPLETION DATE\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.2\", \"Author found\"));\n end\n\n\n\n\nrule \"DOC.6.6: laboratory_project_identification\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Laboratory Project Identification\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\"),\n List.of(\"SPONSOR\", \"VOLUME\", \"This\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.6\", \"Laboratory Project Identification\"));\n end\n\n\n\nrule \"DOC.7.2: study title by document structure\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"STUDY TITLE\" || contains \"Study Title\" || contains \"STUDYTITLE\" || contains \"Report (Final)\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"STUDY TITLE\", \"STUDYTITLE\", \"Report (Final)\"),\n List.of(\"TEST GUIDELINES\", \"TEST GUIDELINE(S)\", \"Guidelines\", \"DATA REQUIREMENT\", \"AUTHOR(S)\", \"AUTHOR\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.7.2\", \"Study title found\"));\n end\n\n\n\nrule \"DOC.8.1: Performing Laboratory\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"PERFORMING LABORATORY\" || contains \"TEST FACILITIES\" || contains \"TEST FACILITY\" || contains \"Test Facility\"), getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Sponsor\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"PERFORMING LABORATORY\", \"TEST FACILITIES\", \"TEST FACILITY\"),\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\", \"Sponsor\", \"PROJECT IDENTIFICATION\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.8.1\", \"Performing Laboratory found\"));\n end\n\n\n\n\nrule \"DOC.8.2: Summary Methods\"\n when\n $headline: Headline(containsString(\"1.1. METHODS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_methods\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.2\", \"Summary Methods found\"));\n end\n\nrule \"DOC.8.3: Summary Observations Laboratory\"\n when\n $headline: Headline(containsString(\"1.2. OBSERVATIONS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_observations\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.3\", \"Summary Observations found\"));\n end\n\n\nrule \"DOC.8.5: Summary Results\"\n when\n Headline((containsStringIgnoreCase(\"1.3. RESULTS\") || containsStringIgnoreCase(\"1.2. RESULTS\")), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> {\n entity.apply(\"DOC.8.5\", \"Summary Results\");\n });\n end\n\nrule \"DOC.8.6: Summary Results 2\"\n when\n $headline: Headline(containsString(\"1.2. RESULTS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\")) \n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.6\", \"Summary Results\"));\n end\n\n\n\nrule \"DOC.8.4: Summary Conclusion\"\n when\n $headline: Headline(containsString(\"1.4. CONCLUSION\") || containsString(\"1.3. CONCLUSION\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_conclusion\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.4\", \"Summary Conculsion found\"));\n end\n\n\n\nrule \"DOC.9.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\")\n || containsString(\"GLP CERTIFICATE (FROM THE COMPETENT AUTHORITY)\")\n || containsString(\"GLP COMPLIANCE STATEMENT\")\n || containsString(\"GLP STATEMENT\") \n )\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.9.0\", \"GLP Study found\");\n });\n end\n\n\n\nrule \"DOC.9.1: GLP Study\"\n when\n $paragraph: Paragraph(containsString(\"GLP COMPLIANCE STATEMENT\"))\n then\n entityCreationService.byRegex(\"GLP COMPLIANCE STATEMENT\", \"glp_study\", EntityType.ENTITY, $paragraph).forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"GLP Study found\");\n });\n end\n\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.addManualChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.addManualChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.addManualChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.addManualChange($legalBasisChange);\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: TextEntity($type: type, $entityType: entityType, !resized(), active())\n $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())\n then\n TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);\n $first.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n $second.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n retract($first);\n retract($second);\n mergedEntity.getIntersectingNodes().forEach(node -> update(node));\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())\n then\n $lowerRank.getIntersectingNodes().forEach(node -> update(node));\n $lowerRank.remove(\"X.6.0\", \"remove Entity of lower rank, when intersected by entity of type ENTITY\");\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt index 1739decc..33456c09 100644 --- a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt +++ b/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt @@ -1 +1 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n//---------------------------------------------------------------------------\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.1.0: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n $page: Page() from $tocHeadline.getParent().getPages()\n $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))\n then\n retract($node);\n end\n\n\n// Rule unit: MAN.0\nrule \"H.2.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.ENTITY);\n end\n\n\nrule \"H.3.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n (containsString(\"DATA REQUIREMENT\") || containsString(\"TEST GUIDELINE\") || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\"))\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock())).flatMap(Collection::stream).findFirst()\n .map(textRange -> $section.getTextBlock().subSequence(textRange).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.1.3: Guidelines\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.2.0: Report number\"\n when\n $section: Section(containsString(\"LABORATORY PROJECT ID\") , containsString(\"Report Number:\"))\n then\n entityCreationService.lineAfterString(\"Report Number:\", \"report_number\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.2.0\", \"Report number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.3.0: Experimental Starting Date\"\n when\n $section: Section(containsString(\"Experimental I. Starting Date:\") || containsString(\"Experimental II. Starting Date:\") || containsStringIgnoreCase(\"experimental start date\") || containsStringIgnoreCase(\"experimental starting date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental start date\",\n \"Experimental start date:\",\n \"Experimental Starting Date\",\n \"Experimental Starting Date:\",\n \"Experimental starting date\",\n \"Experimental starting date:\",\n \"Experimental Start Date\",\n \"Experimental Start Date:\",\n \"Experimental I. Starting Date:\",\n \"Experimental II. Starting Date:\"), \"experimental_start_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.3.0\", \"Experimental start date found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.4.0: Experimental Completion Date\"\n when\n $section: Section(containsStringIgnoreCase(\"experimental termination date\") || containsStringIgnoreCase(\"experimental completion date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental termination date\",\n \"Experimental termination date:\",\n \"Experimental Completion Date\",\n \"Experimental Completion Date:\",\n \"Experimental completion date\",\n \"Experimental completion date:\",\n \"Experimental Termination Date\",\n \"Experimental Termination Date:\"), \"experimental_end_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.4.0\", \"Experimental end date found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.5.0: Ignore species and strain in irrelevant study types\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"471\",\"474\",\"487\"))\n $section: Section(hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\n rule \"DOC.5.1: Hide all skipped species and strains except in the relevant sections\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.5.2: Species\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.5.2\", \"Species found.\", \"n-a\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.3: Strain\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.3\", \"Strain found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.0: study title by document structure\"\n when\n $table: Table(onPage(1),\n (containsString(\"Final Report\") || containsString(\"SPL\")),\n numberOfRows == 1,\n numberOfCols == 1)\n then\n\n entityCreationService.bySemanticNode($table.getCell(0, 0).streamChildren().toList().get(1), \"title\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.6.0\", \"Study title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.1: study title\"\n when\n $table: Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $table).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.1\", \"Title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.2: study title\"\n when\n not Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n $section: Section(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.2\", \"Title found\", \"n-a\");\n });\n end\n\n\n\nrule \"DOC.7.0: Performing Laboratory (Name)\"\n when\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n entityCreationService.lineAfterString(\"PERFORMING LABORATORY:\", \"laboratory_name\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.7.0\", \"Performing Laboratory found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.7.1: Performing Laboratory (Country)\"\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"COUNTRY\"))\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n nerEntities.streamEntitiesOfType(\"COUNTRY\")\n .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, \"laboratory_country\", EntityType.ENTITY, $section))\n .forEach(entity -> {\n entity.apply(\"DOC.7.1\", \"Performing Laboratory found\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.7.2: Performing Laboratory (Country & Name) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && (containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(\"laboratory_country\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory country dictionary entry found.\", \"n-a\");\n });\n $section.getEntitiesOfType(\"laboratory_name\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory name dictionary entry found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.7.3: Performing Laboratory (Country) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && !(containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(List.of(\"laboratory_country\", \"laboratory_name\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.8.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\"))\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.8.0\", \"GLP Study found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.0: Batch number from CoA\"\n when\n $section: Section(\n (\n anyHeadlineContainsString(\"Analytical Report\")\n || anyHeadlineContainsStringIgnoreCase(\"Certificate of Analysis\")\n || containsStringIgnoreCase(\"Certificate of Analysis\")\n )\n && (\n containsStringIgnoreCase(\"batch\")\n || containsStringIgnoreCase(\"bath\")\n || containsStringIgnoreCase(\"barch\")\n || containsStringIgnoreCase(\"bateb\")\n )\n && (\n containsStringIgnoreCase(\"identification\")\n || containsStringIgnoreCase(\"ldentitfication\")\n || containsStringIgnoreCase(\"wentification\")\n || containsStringIgnoreCase(\"mentification\")\n || containsStringIgnoreCase(\"kientification\")\n || containsStringIgnoreCase(\"reference number\")\n || containsStringIgnoreCase(\"test substance\")\n )\n )\n then\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"(Batch Identification):\",\n \"Bateb Identification\",\n \"Batch Wentification\",\n \"Batch Mentification\",\n \"Batch Kientification\",\n \"Barch Identification\",\n \"Bath ldentitfication\",\n \"Batch of test substance :\"), \"batch_number\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.9.0\", \"Batch number found in CoA\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.1: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n then\n Stream.of(entityCreationService.byRegex(\"Batch ID ([A-Z\\\\d\\\\-]{7,14})\", \"batch_number\", EntityType.ENTITY, 1, $section),\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\"\n ), \"batch_number\", EntityType.ENTITY, $section)).flatMap(a -> a)\n .forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"Batch number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.2: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n $batchNumber: String() from List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\")\n $table: Table(containsStringIgnoreCase($batchNumber)) from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()\n then\n entityCreationService.lineAfterStringAcrossColumnsIgnoreCase($batchNumber, \"batch_number\", EntityType.ENTITY, $table).forEach(entity -> {\n entity.apply(\"DOC.9.2\", \"Batch number found\", \"n-a\");\n });\n end\n\n\n\n\nrule \"DOC.10.0: Conclusions - LD50, LC50, Confidence\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"425\",\"436\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"Conclusion\") || anyHeadlineContainsStringIgnoreCase(\"Lethality\"))\n && (containsString(\"LD\") || containsString(\"LC\") || containsString(\"50\") || containsString(\"LD50\") || containsString(\"lethal concentration\") || containsString(\"lethal dose\"))\n && (\n containsString(\"greater than\")\n || containsString(\"higher than\")\n || containsString(\"above\")\n || containsString(\"in excess\")\n || containsString(\"exceeds\")\n || containsString(\"was found to be\")\n || containsString(\"was calculated to be\")\n || containsString(\"estimated to be\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(L[D|C]\\\\s?50|lethal concentration|lethal dose).{1,200}(greater than|considered to be above|in excess of|exceeds|higher than)\", \"ld50_greater\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 greater than found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"\\\\b(?:(?:greater|higher) than|considered to be above|(?:was|is) (?:found|estimated) to be|was calculated to be|in excess of|exceeds|equal to)\\\\s?([\\\\d\\\\.]{1,6})\\\\s?mg\\\\/(?:kg|L)\", \"ld50_value\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 value found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_minimal\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Minimal Confidence found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_maximal\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Maximal Confidence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.0: Guideline Deviation\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"General Information\") || containsString(\"GENERAL INFORMATION\"))\n && (containsStringIgnoreCase(\"from the\") || containsStringIgnoreCase(\"to the\"))\n )\n then\n entityCreationService.betweenRegexes(\"(?:Deviations? from the [G|g]uidelines?)(?: and| or)?( the)?(?: Study Plan)?\", \"(?:(?:Deviations? from the Study Plan)|(?:Performing laboratory test)|(?:Other)|(?:Retention of [S|s]amples)|(?:Amendments? to Final Protocol))\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Guidelines found\", \"n-a\");\n });\n entityCreationService.betweenRegexes(\"(?:Deviations? (?:from|to)(?: the)? [S|s]tudy [P|p]lan)\", \"(?:Regulatory Guidelines)|(?:Other)|(?:Distribution of the report)|(?:Performing laboratory test)|(?:Distribution of the report)|(?:Retention of [S|s]amples)\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Study Plan found\", \"n-a\");\n });\n entityCreationService.betweenStrings(\"Deviations from the study plan\", \"Regulatory Guidelines\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?>Study plan adherence)(.{1,20}deviations.{1,20} to the study plan.{0,50}\\\\.)\\\\s\", \"guideline_deviation\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Guideline deviation found in text.\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeEnd(\"Deviations from the study plan\", \"validity of the study.\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.1: Guideline Deviation in text\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Introduction\")\n && containsStringIgnoreCase(\"deviations from the protocol\")\n )\n then\n entityCreationService.byRegex(\"There were no deviations from the protocol.{1,100}\\\\.\\\\s\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.1\", \"Guideline deviation found in text.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.12.0: Clinical Signs\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Clinical Signs\", \"Macroscopic Findings\") && !containsString(\"TABLE\") && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"clinical_signs\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.12.0\", \"Clinical Signs found\", \"n-a\"));\n end\n\n\nrule \"DOC.13.0: Dosages\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $section: Section(\n (anyHeadlineContainsStringIgnoreCase(\"Dosages\") || anyHeadlineContainsStringIgnoreCase(\"Study Design\"))\n && !getHeadline().containsString(\"TABLE\")\n )\n then\n entityCreationService.betweenStringsIncludeStartAndEnd(\"The animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeStartAndEnd(\"Animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.byRegexWithLineBreaks(\"(?:\\\\.[\\\\s|\\\\n]|^.{5,20}\\\\n)([^\\\\.]{1,200}(?:animal|given|received)[^\\\\.]{1,200}dose\\\\s(?:levels?\\\\s)?(?:of|at)[^\\\\.]{1,200})(?:\\\\.[\\\\s|\\\\n|$])\", \"dosages\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.14.0: Mortality\"\n when\n $headline: Headline(containsString(\"Mortality\") && !containsString(\"TABLE\"))\n FileAttribute(label == \"OECD Number\", value == \"425\")\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.14.0\", \"Mortality found\", \"n-a\"));\n end\n\n\nrule \"DOC.15.0: Study Conclusion\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Conclusion\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.15.0\", \"Study Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.16.0: Weight Behavior Changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && (\n containsString(\"body weight\")\n || containsString(\"body weights\")\n || containsString(\"bodyweight\")\n || containsString(\"bodyweights\")\n )\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"weight_behavior_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.16.0\", \"Weight behavior changes found\", \"n-a\"));\n end\n\n\nrule \"DOC.17.0: Necropsy findings\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"436\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Necropsy\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic Findings\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic examination\")\n )\n && !getHeadline().containsStringIgnoreCase(\"Table\")\n && !getHeadline().containsStringIgnoreCase(\"Appendix\")\n && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"necropsy_findings\", EntityType.ENTITY)\n .forEach( entity -> entity.apply(\"DOC.17.0\", \"Necropsy section found\", \"n-a\"));\n end\n\n\nrule \"DOC.18.0: Clinical observations\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Clinical Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Clinical observations\")\n || anyHeadlineContainsStringIgnoreCase(\"In-life Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Postmortem Observations\")\n )\n && !anyHeadlineContainsStringIgnoreCase(\"Appendix\")\n && !anyHeadlineContainsStringIgnoreCase(\"Table\")\n && !anyHeadlineContainsStringIgnoreCase(\"Mortality\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"clinical_observations\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.18.0\", \"Clinical observations section found\", \"n-a\"));\n end\n\n\nrule \"DOC.19.0: Bodyweight changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Bodyweight\", \"Bodyweights\", \"Body Weights\", \"Body Weight\"), !containsAnyStringIgnoreCase(\"Appendix\", \"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"bodyweight_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.19.0\", \"Bodyweight section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.0: Study Design\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"404\",\"405\",\"406\",\"428\",\"429\",\"438\",\"439\",\"474\",\"487\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n && !anyHeadlineContainsString(\"Preliminary screening test\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_design\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.20.0\", \"Study design section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.1: Study Design\"\n when\n Headline(containsStringIgnoreCase(\"Study Design\"), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_design\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.20.1\", \"Study design section found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.21.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n $parentHeadline: Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n not Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($parentHeadline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.0\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.21.1: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.1\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.22.0: Detailing (404 & 405)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"404\",\"405\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"Results\")\n && !getHeadline().containsStringIgnoreCase(\"Evaluation\")\n && !getHeadline().containsStringIgnoreCase(\"study\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"detailing\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.22.0\", \"Detailing found\", \"n-a\"));\n end\n\n\nrule \"DOC.23.0: Preliminary Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n ((anyHeadlineContainsString(\"Preliminary Screening Test\") && containsString(\"Clinical observations\"))\n || anyHeadlineContainsString(\"Pre-Experiment\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"preliminary_test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.23.0\", \"Preliminary Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.0: Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section((getHeadline().containsString(\"RESULTS AND DISCUSSION\") || getHeadline().containsString(\"Estimation of the proliferative response of lymph node cells\") || getHeadline().containsString(\"Results in the Main Experiment\")))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.24.0\", \"Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.1: Test Results (429)\"\n when\n Headline(containsStringIgnoreCase(\"RESULTS AND DISCUSSION\"), $sectionIdentifierResultsAndDiscussion: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifierResultsAndDiscussion))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"test_results\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.24.1\", \"Test Results found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.25.0: Approach used (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n hasEntitiesOfType(\"species\")\n && (containsStringIgnoreCase(\"animals per\") || containsStringIgnoreCase(\"animals /\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\banimals (?:per|\\\\/) .{0,15}(group)\\\\b\", \"approach_used\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.25.0\", \"Study animal approach found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.26.0: Sex\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"405\",\"429\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.26.0\", \"Test animal sex found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.27.0: Animal Number 405\"\n when\n FileAttribute(label == \"OECD Number\", value == \"405\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"reaction\")\n )\n && !getHeadline().containsString(\"selection\")\n && (\n containsStringIgnoreCase(\"number of animals\")\n || containsStringIgnoreCase(\"no.\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?:.{1,10} No\\\\. )([\\\\d\\\\w\\\\-]{3,8})\", \"animal_numbers\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.0: Animal Number 429\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsString(\"selection\")\n && containsStringIgnoreCase(\"number of animals\")\n && (containsStringIgnoreCase(\"per\") || containsString(\"/\"))\n && containsStringIgnoreCase(\"group\")\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:).{0,60}\\\\b([\\\\d]{1,3})\\\\sper group\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"([\\\\d]{1,3})[\\\\w\\\\s\\\\/]{0,20}(?:treatment )?group\\\\b\", \"number_of_animals\", EntityType.ENTITY,1 , $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.1: No. Of animals - Fallback to appendix tables listing all individual animals for 429\"\n when\n $keyword: String() from List.of(\"Animal Number\", \"Animal No.\", \"Animal number\")\n $table: Table(containsString($keyword) && getHeadline().containsString(\"TABLE\") && getHeadline().containsString(\"Individual\"))\n FileAttribute(label == \"OECD Number\", value == \"429\")\n then\n $table.streamTableCellsWithHeader($keyword)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"animal_numbers\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.28.1\", \"Animal number found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.29.0: 4h Exposure\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"403\",\"436\"))\n $section: Section(\n (containsStringIgnoreCase(\"4 hours\") || containsStringIgnoreCase(\"four hours\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"((?<=\\\\.\\\\s\\\\b).{1,100}(4|four) hours.*?\\\\.) \", \"4h_exposure\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.29.0\", \"4h exposure sentence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.30.0: Dilution of the test substance\"\n when\n FileAttribute(label == \"OECD Number\", value == \"404\")\n $section: Section(\n getHeadline().containsString(\"Formulation\")\n && containsString(\"dilution\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"dilution\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.30.0\", \"Dilution found.\", \"n-a\"));\n end\n\n\nrule \"DOC.31.0: Positive Control\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Positive Control\")\n && !(getHeadline().containsStringIgnoreCase(\"Appendix\") || getHeadline().containsStringIgnoreCase(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"positive_control\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.31.0\", \"Positive control found.\", \"n-a\"));\n end\n\n\nrule \"DOC.32.0: Mortality Statement\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $headline: Headline(containsStringIgnoreCase(\"Mortality\") && !containsString(\"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality_statement\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.32.0\", \"Mortality Statement found\", \"n-a\"));\n end\n\n\nrule \"DOC.33.0: Dose Mortality\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $table: Table(\n (hasHeader(\"Mortality\") || hasHeader(\"Long Term Results\") || hasHeader(\"LongTerm Outcome\") || hasHeader(\"Long Term Outcome\") || hasHeader(\"Comments\") || hasHeader(\"Viability / Mortality\") || hasHeader(\"Viability/Mortality\"))\n &&\n (hasHeader(\"Dose [mg/kg bodyweight]\") || hasHeader(\"Dose [mg/kg body weight]\") ||hasHeader(\"Dose (mg/kg)\") || hasHeader(\"Dose levei (mg/kg)\") || hasHeader(\"Dose Level (mg/kg)\") || hasHeader(\"Dose level (mg/kg)\") || hasHeader(\"Dosage [mg/kg body weight]\"))\n )\n then\n Stream.of($table.streamTableCellsWithHeader(\"Mortality\"),\n $table.streamTableCellsWithHeader(\"Comments\"),\n $table.streamTableCellsWithHeader(\"Long Term Results\"),\n $table.streamTableCellsWithHeader(\"Long Term Outcome\"),\n $table.streamTableCellsWithHeader(\"LongTerm Outcome\"),\n $table.streamTableCellsWithHeader(\"Viability / Mortality\"),\n $table.streamTableCellsWithHeader(\"Viability/Mortality\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n\n Stream.of($table.streamTableCellsWithHeader(\"Dose [mg/kg bodyweight]\"),\n $table.streamTableCellsWithHeader(\"Dose [mg/kg body weight]\"),\n $table.streamTableCellsWithHeader(\"Dose levei (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose Level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dosage [mg/kg body weight]\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality_dose\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.34.0: Results (Main Study)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && getHeadline().getTextRange().length() < 20\n && !(getHeadline().containsString(\"Appendix\") || getHeadline().containsString(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"results_(main_study)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.34.0\", \"Results for main study found.\", \"n-a\"));\n end\n\n\nrule \"DOC.35.0: Doses (mg/kg bodyweight)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"doses_(mg_kg_bw)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.35.0\", \"Doses per bodyweight information found\", \"n-a\"));\n end\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.getManualOverwrite().addChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);\n end\n\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.7\nrule \"X.7.0: remove all images\"\n salience 512\n when\n $image: Image(imageType != ImageType.OCR, !hasManualChanges())\n then\n $image.remove(\"X.7.0\", \"remove all images\");\n retract($image);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file +"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n//---------------------------------------------------------------------------\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.1.0: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n $page: Page() from $tocHeadline.getParent().getPages()\n $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))\n then\n retract($node);\n end\n\n\n// Rule unit: MAN.0\nrule \"H.2.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.ENTITY);\n end\n\n\nrule \"H.3.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n (containsString(\"DATA REQUIREMENT\") || containsString(\"TEST GUIDELINE\") || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\"))\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock())).flatMap(Collection::stream).findFirst()\n .map(textRange -> $section.getTextBlock().subSequence(textRange).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.1.3: Guidelines\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.2.0: Report number\"\n when\n $section: Section(containsString(\"LABORATORY PROJECT ID\") , containsString(\"Report Number:\"))\n then\n entityCreationService.lineAfterString(\"Report Number:\", \"report_number\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.2.0\", \"Report number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.3.0: Experimental Starting Date\"\n when\n $section: Section(containsString(\"Experimental I. Starting Date:\") || containsString(\"Experimental II. Starting Date:\") || containsStringIgnoreCase(\"experimental start date\") || containsStringIgnoreCase(\"experimental starting date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental start date\",\n \"Experimental start date:\",\n \"Experimental Starting Date\",\n \"Experimental Starting Date:\",\n \"Experimental starting date\",\n \"Experimental starting date:\",\n \"Experimental Start Date\",\n \"Experimental Start Date:\",\n \"Experimental I. Starting Date:\",\n \"Experimental II. Starting Date:\"), \"experimental_start_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.3.0\", \"Experimental start date found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.4.0: Experimental Completion Date\"\n when\n $section: Section(containsStringIgnoreCase(\"experimental termination date\") || containsStringIgnoreCase(\"experimental completion date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental termination date\",\n \"Experimental termination date:\",\n \"Experimental Completion Date\",\n \"Experimental Completion Date:\",\n \"Experimental completion date\",\n \"Experimental completion date:\",\n \"Experimental Termination Date\",\n \"Experimental Termination Date:\"), \"experimental_end_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.4.0\", \"Experimental end date found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.5.0: Ignore species and strain in irrelevant study types\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"471\",\"474\",\"487\"))\n $section: Section(hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\n rule \"DOC.5.1: Hide all skipped species and strains except in the relevant sections\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.5.2: Species\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.5.2\", \"Species found.\", \"n-a\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.3: Strain\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.3\", \"Strain found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.0: study title by document structure\"\n when\n $table: Table(onPage(1),\n (containsString(\"Final Report\") || containsString(\"SPL\")),\n numberOfRows == 1,\n numberOfCols == 1)\n then\n\n entityCreationService.bySemanticNode($table.getCell(0, 0).streamChildren().toList().get(1), \"title\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.6.0\", \"Study title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.1: study title\"\n when\n $table: Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $table).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.1\", \"Title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.2: study title\"\n when\n not Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n $section: Section(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.2\", \"Title found\", \"n-a\");\n });\n end\n\n\n\nrule \"DOC.7.0: Performing Laboratory (Name)\"\n when\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n entityCreationService.lineAfterString(\"PERFORMING LABORATORY:\", \"laboratory_name\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.7.0\", \"Performing Laboratory found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.7.1: Performing Laboratory (Country)\"\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"COUNTRY\"))\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n nerEntities.streamEntitiesOfType(\"COUNTRY\")\n .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, \"laboratory_country\", EntityType.ENTITY, $section))\n .forEach(entity -> {\n entity.apply(\"DOC.7.1\", \"Performing Laboratory found\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.7.2: Performing Laboratory (Country & Name) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && (containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(\"laboratory_country\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory country dictionary entry found.\", \"n-a\");\n });\n $section.getEntitiesOfType(\"laboratory_name\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory name dictionary entry found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.7.3: Performing Laboratory (Country) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && !(containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(List.of(\"laboratory_country\", \"laboratory_name\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.8.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\"))\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.8.0\", \"GLP Study found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.0: Batch number from CoA\"\n when\n $section: Section(\n (\n anyHeadlineContainsString(\"Analytical Report\")\n || anyHeadlineContainsStringIgnoreCase(\"Certificate of Analysis\")\n || containsStringIgnoreCase(\"Certificate of Analysis\")\n )\n && (\n containsStringIgnoreCase(\"batch\")\n || containsStringIgnoreCase(\"bath\")\n || containsStringIgnoreCase(\"barch\")\n || containsStringIgnoreCase(\"bateb\")\n )\n && (\n containsStringIgnoreCase(\"identification\")\n || containsStringIgnoreCase(\"ldentitfication\")\n || containsStringIgnoreCase(\"wentification\")\n || containsStringIgnoreCase(\"mentification\")\n || containsStringIgnoreCase(\"kientification\")\n || containsStringIgnoreCase(\"reference number\")\n || containsStringIgnoreCase(\"test substance\")\n )\n )\n then\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"(Batch Identification):\",\n \"Bateb Identification\",\n \"Batch Wentification\",\n \"Batch Mentification\",\n \"Batch Kientification\",\n \"Barch Identification\",\n \"Bath ldentitfication\",\n \"Batch of test substance :\"), \"batch_number\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.9.0\", \"Batch number found in CoA\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.1: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n then\n Stream.of(entityCreationService.byRegex(\"Batch ID ([A-Z\\\\d\\\\-]{7,14})\", \"batch_number\", EntityType.ENTITY, 1, $section),\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\"\n ), \"batch_number\", EntityType.ENTITY, $section)).flatMap(a -> a)\n .forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"Batch number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.2: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n $batchNumber: String() from List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\")\n $table: Table(containsStringIgnoreCase($batchNumber)) from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()\n then\n entityCreationService.lineAfterStringAcrossColumnsIgnoreCase($batchNumber, \"batch_number\", EntityType.ENTITY, $table).forEach(entity -> {\n entity.apply(\"DOC.9.2\", \"Batch number found\", \"n-a\");\n });\n end\n\n\n\n\nrule \"DOC.10.0: Conclusions - LD50, LC50, Confidence\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"425\",\"436\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"Conclusion\") || anyHeadlineContainsStringIgnoreCase(\"Lethality\"))\n && (containsString(\"LD\") || containsString(\"LC\") || containsString(\"50\") || containsString(\"LD50\") || containsString(\"lethal concentration\") || containsString(\"lethal dose\"))\n && (\n containsString(\"greater than\")\n || containsString(\"higher than\")\n || containsString(\"above\")\n || containsString(\"in excess\")\n || containsString(\"exceeds\")\n || containsString(\"was found to be\")\n || containsString(\"was calculated to be\")\n || containsString(\"estimated to be\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(L[D|C]\\\\s?50|lethal concentration|lethal dose).{1,200}(greater than|considered to be above|in excess of|exceeds|higher than)\", \"ld50_greater\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 greater than found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"\\\\b(?:(?:greater|higher) than|considered to be above|(?:was|is) (?:found|estimated) to be|was calculated to be|in excess of|exceeds|equal to)\\\\s?([\\\\d\\\\.]{1,6})\\\\s?mg\\\\/(?:kg|L)\", \"ld50_value\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 value found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_minimal\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Minimal Confidence found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_maximal\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Maximal Confidence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.0: Guideline Deviation\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"General Information\") || containsString(\"GENERAL INFORMATION\"))\n && (containsStringIgnoreCase(\"from the\") || containsStringIgnoreCase(\"to the\"))\n )\n then\n entityCreationService.betweenRegexes(\"(?:Deviations? from the [G|g]uidelines?)(?: and| or)?( the)?(?: Study Plan)?\", \"(?:(?:Deviations? from the Study Plan)|(?:Performing laboratory test)|(?:Other)|(?:Retention of [S|s]amples)|(?:Amendments? to Final Protocol))\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Guidelines found\", \"n-a\");\n });\n entityCreationService.betweenRegexes(\"(?:Deviations? (?:from|to)(?: the)? [S|s]tudy [P|p]lan)\", \"(?:Regulatory Guidelines)|(?:Other)|(?:Distribution of the report)|(?:Performing laboratory test)|(?:Distribution of the report)|(?:Retention of [S|s]amples)\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Study Plan found\", \"n-a\");\n });\n entityCreationService.betweenStrings(\"Deviations from the study plan\", \"Regulatory Guidelines\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?>Study plan adherence)(.{1,20}deviations.{1,20} to the study plan.{0,50}\\\\.)\\\\s\", \"guideline_deviation\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Guideline deviation found in text.\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeEnd(\"Deviations from the study plan\", \"validity of the study.\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.1: Guideline Deviation in text\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Introduction\")\n && containsStringIgnoreCase(\"deviations from the protocol\")\n )\n then\n entityCreationService.byRegex(\"There were no deviations from the protocol.{1,100}\\\\.\\\\s\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.1\", \"Guideline deviation found in text.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.12.0: Clinical Signs\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Clinical Signs\", \"Macroscopic Findings\") && !containsString(\"TABLE\") && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"clinical_signs\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.12.0\", \"Clinical Signs found\", \"n-a\"));\n end\n\n\nrule \"DOC.13.0: Dosages\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $section: Section(\n (anyHeadlineContainsStringIgnoreCase(\"Dosages\") || anyHeadlineContainsStringIgnoreCase(\"Study Design\"))\n && !getHeadline().containsString(\"TABLE\")\n )\n then\n entityCreationService.betweenStringsIncludeStartAndEnd(\"The animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeStartAndEnd(\"Animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.byRegexWithLineBreaks(\"(?:\\\\.[\\\\s|\\\\n]|^.{5,20}\\\\n)([^\\\\.]{1,200}(?:animal|given|received)[^\\\\.]{1,200}dose\\\\s(?:levels?\\\\s)?(?:of|at)[^\\\\.]{1,200})(?:\\\\.[\\\\s|\\\\n|$])\", \"dosages\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.14.0: Mortality\"\n when\n $headline: Headline(containsString(\"Mortality\") && !containsString(\"TABLE\"))\n FileAttribute(label == \"OECD Number\", value == \"425\")\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.14.0\", \"Mortality found\", \"n-a\"));\n end\n\n\nrule \"DOC.15.0: Study Conclusion\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Conclusion\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.15.0\", \"Study Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.16.0: Weight Behavior Changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && (\n containsString(\"body weight\")\n || containsString(\"body weights\")\n || containsString(\"bodyweight\")\n || containsString(\"bodyweights\")\n )\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"weight_behavior_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.16.0\", \"Weight behavior changes found\", \"n-a\"));\n end\n\n\nrule \"DOC.17.0: Necropsy findings\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"436\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Necropsy\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic Findings\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic examination\")\n )\n && !getHeadline().containsStringIgnoreCase(\"Table\")\n && !getHeadline().containsStringIgnoreCase(\"Appendix\")\n && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"necropsy_findings\", EntityType.ENTITY)\n .forEach( entity -> entity.apply(\"DOC.17.0\", \"Necropsy section found\", \"n-a\"));\n end\n\n\nrule \"DOC.18.0: Clinical observations\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Clinical Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Clinical observations\")\n || anyHeadlineContainsStringIgnoreCase(\"In-life Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Postmortem Observations\")\n )\n && !anyHeadlineContainsStringIgnoreCase(\"Appendix\")\n && !anyHeadlineContainsStringIgnoreCase(\"Table\")\n && !anyHeadlineContainsStringIgnoreCase(\"Mortality\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"clinical_observations\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.18.0\", \"Clinical observations section found\", \"n-a\"));\n end\n\n\nrule \"DOC.19.0: Bodyweight changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Bodyweight\", \"Bodyweights\", \"Body Weights\", \"Body Weight\"), !containsAnyStringIgnoreCase(\"Appendix\", \"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"bodyweight_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.19.0\", \"Bodyweight section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.0: Study Design\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"404\",\"405\",\"406\",\"428\",\"429\",\"438\",\"439\",\"474\",\"487\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n && !anyHeadlineContainsString(\"Preliminary screening test\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_design\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.20.0\", \"Study design section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.1: Study Design\"\n when\n Headline(containsStringIgnoreCase(\"Study Design\"), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_design\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.20.1\", \"Study design section found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.21.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n $parentHeadline: Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n not Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($parentHeadline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.0\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.21.1: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.1\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.22.0: Detailing (404 & 405)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"404\",\"405\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"Results\")\n && !getHeadline().containsStringIgnoreCase(\"Evaluation\")\n && !getHeadline().containsStringIgnoreCase(\"study\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"detailing\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.22.0\", \"Detailing found\", \"n-a\"));\n end\n\n\nrule \"DOC.23.0: Preliminary Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n ((anyHeadlineContainsString(\"Preliminary Screening Test\") && containsString(\"Clinical observations\"))\n || anyHeadlineContainsString(\"Pre-Experiment\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"preliminary_test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.23.0\", \"Preliminary Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.0: Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section((getHeadline().containsString(\"RESULTS AND DISCUSSION\") || getHeadline().containsString(\"Estimation of the proliferative response of lymph node cells\") || getHeadline().containsString(\"Results in the Main Experiment\")))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.24.0\", \"Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.1: Test Results (429)\"\n when\n Headline(containsStringIgnoreCase(\"RESULTS AND DISCUSSION\"), $sectionIdentifierResultsAndDiscussion: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifierResultsAndDiscussion))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"test_results\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.24.1\", \"Test Results found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.25.0: Approach used (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n hasEntitiesOfType(\"species\")\n && (containsStringIgnoreCase(\"animals per\") || containsStringIgnoreCase(\"animals /\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\banimals (?:per|\\\\/) .{0,15}(group)\\\\b\", \"approach_used\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.25.0\", \"Study animal approach found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.26.0: Sex\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"405\",\"429\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.26.0\", \"Test animal sex found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.27.0: Animal Number 405\"\n when\n FileAttribute(label == \"OECD Number\", value == \"405\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"reaction\")\n )\n && !getHeadline().containsString(\"selection\")\n && (\n containsStringIgnoreCase(\"number of animals\")\n || containsStringIgnoreCase(\"no.\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?:.{1,10} No\\\\. )([\\\\d\\\\w\\\\-]{3,8})\", \"animal_numbers\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.0: Animal Number 429\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsString(\"selection\")\n && containsStringIgnoreCase(\"number of animals\")\n && (containsStringIgnoreCase(\"per\") || containsString(\"/\"))\n && containsStringIgnoreCase(\"group\")\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:).{0,60}\\\\b([\\\\d]{1,3})\\\\sper group\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"([\\\\d]{1,3})[\\\\w\\\\s\\\\/]{0,20}(?:treatment )?group\\\\b\", \"number_of_animals\", EntityType.ENTITY,1 , $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.1: No. Of animals - Fallback to appendix tables listing all individual animals for 429\"\n when\n $keyword: String() from List.of(\"Animal Number\", \"Animal No.\", \"Animal number\")\n $table: Table(containsString($keyword) && getHeadline().containsString(\"TABLE\") && getHeadline().containsString(\"Individual\"))\n FileAttribute(label == \"OECD Number\", value == \"429\")\n then\n $table.streamTableCellsWithHeader($keyword)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"animal_numbers\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.28.1\", \"Animal number found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.29.0: 4h Exposure\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"403\",\"436\"))\n $section: Section(\n (containsStringIgnoreCase(\"4 hours\") || containsStringIgnoreCase(\"four hours\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"((?<=\\\\.\\\\s\\\\b).{1,100}(4|four) hours.*?\\\\.) \", \"4h_exposure\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.29.0\", \"4h exposure sentence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.30.0: Dilution of the test substance\"\n when\n FileAttribute(label == \"OECD Number\", value == \"404\")\n $section: Section(\n getHeadline().containsString(\"Formulation\")\n && containsString(\"dilution\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"dilution\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.30.0\", \"Dilution found.\", \"n-a\"));\n end\n\n\nrule \"DOC.31.0: Positive Control\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Positive Control\")\n && !(getHeadline().containsStringIgnoreCase(\"Appendix\") || getHeadline().containsStringIgnoreCase(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"positive_control\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.31.0\", \"Positive control found.\", \"n-a\"));\n end\n\n\nrule \"DOC.32.0: Mortality Statement\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $headline: Headline(containsStringIgnoreCase(\"Mortality\") && !containsString(\"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality_statement\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.32.0\", \"Mortality Statement found\", \"n-a\"));\n end\n\n\nrule \"DOC.33.0: Dose Mortality\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $table: Table(\n (hasHeader(\"Mortality\") || hasHeader(\"Long Term Results\") || hasHeader(\"LongTerm Outcome\") || hasHeader(\"Long Term Outcome\") || hasHeader(\"Comments\") || hasHeader(\"Viability / Mortality\") || hasHeader(\"Viability/Mortality\"))\n &&\n (hasHeader(\"Dose [mg/kg bodyweight]\") || hasHeader(\"Dose [mg/kg body weight]\") ||hasHeader(\"Dose (mg/kg)\") || hasHeader(\"Dose levei (mg/kg)\") || hasHeader(\"Dose Level (mg/kg)\") || hasHeader(\"Dose level (mg/kg)\") || hasHeader(\"Dosage [mg/kg body weight]\"))\n )\n then\n Stream.of($table.streamTableCellsWithHeader(\"Mortality\"),\n $table.streamTableCellsWithHeader(\"Comments\"),\n $table.streamTableCellsWithHeader(\"Long Term Results\"),\n $table.streamTableCellsWithHeader(\"Long Term Outcome\"),\n $table.streamTableCellsWithHeader(\"LongTerm Outcome\"),\n $table.streamTableCellsWithHeader(\"Viability / Mortality\"),\n $table.streamTableCellsWithHeader(\"Viability/Mortality\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n\n Stream.of($table.streamTableCellsWithHeader(\"Dose [mg/kg bodyweight]\"),\n $table.streamTableCellsWithHeader(\"Dose [mg/kg body weight]\"),\n $table.streamTableCellsWithHeader(\"Dose levei (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose Level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dosage [mg/kg body weight]\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality_dose\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.34.0: Results (Main Study)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && getHeadline().getTextRange().length() < 20\n && !(getHeadline().containsString(\"Appendix\") || getHeadline().containsString(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"results_(main_study)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.34.0\", \"Results for main study found.\", \"n-a\"));\n end\n\n\nrule \"DOC.35.0: Doses (mg/kg bodyweight)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"doses_(mg_kg_bw)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.35.0\", \"Doses per bodyweight information found\", \"n-a\"));\n end\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.addManualChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.addManualChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.addManualChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.addManualChange($legalBasisChange);\n end\n\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.7\nrule \"X.7.0: remove all images\"\n salience 512\n when\n $image: Image(imageType != ImageType.OCR, !hasManualChanges())\n then\n $image.remove(\"X.7.0\", \"remove all images\");\n retract($image);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file