diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java index 5c7ba341..faf1de14 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java @@ -9,6 +9,8 @@ import java.util.List; import java.util.Optional; import java.util.stream.Stream; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.GenericSemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; @@ -17,6 +19,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.utils.EntityCreationUtility; +import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -35,7 +39,7 @@ public class DocumentTree { public DocumentTree(Document document) { - root = Entry.builder().treeId(Collections.emptyList()).children(new LinkedList<>()).node(document).build(); + this.root = Entry.builder().treeId(Collections.emptyList()).children(new LinkedList<>()).node(document).build(); } @@ -358,6 +362,25 @@ public class DocumentTree { } + public void addEntityToGraph(TextEntity entity) { + + getRoot().getNode().addThisToEntityIfIntersects(entity); + + TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock(); + EntityEnrichmentService.enrichEntity(entity, textBlock); + + EntityCreationUtility.addToPages(entity); + EntityCreationUtility.addEntityToNodeEntitySets(entity); + + if (entity.getEntityType().equals(EntityType.TEMPORARY)) { + return; + } + + entity.computeRelations(); + entity.notifyEntityInserted(); + } + + @Builder @Getter @AllArgsConstructor diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/IntersectingNodeVisitor.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/IntersectingNodeVisitor.java new file mode 100644 index 00000000..d0ec319e --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/IntersectingNodeVisitor.java @@ -0,0 +1,32 @@ +package com.iqser.red.service.redaction.v1.server.model.document; + +import java.util.HashSet; +import java.util.Set; + +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; + +import lombok.Getter; + +public class IntersectingNodeVisitor extends AbstractNodeVisitor { + + @Getter + private Set intersectingNodes; + private final TextRange textRange; + + + public IntersectingNodeVisitor(TextRange textRange) { + + this.textRange = textRange; + this.intersectingNodes = new HashSet<>(); + } + + + @Override + public void visitNodeDefault(SemanticNode node) { + + if (textRange.intersects(node.getTextRange())) { + intersectingNodes.add(node); + } + } + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractRelation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractRelation.java new file mode 100644 index 00000000..97b3d90e --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractRelation.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public abstract class AbstractRelation implements Relation { + + protected final TextEntity a; + protected final TextEntity b; + + + @Override + public String toString() { + + return this.getClass().getSimpleName() + "{" + "a=" + a + ", b=" + b + '}'; + } + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java new file mode 100644 index 00000000..d3ac61f1 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public class Containment extends Intersection { + + public Containment(TextEntity container, TextEntity contained) { + + super(container, contained); + } + + public TextEntity getContainer() { + return a; + } + + public TextEntity getContained() { + return b; + } + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityEventListener.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityEventListener.java new file mode 100644 index 00000000..10f2a136 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityEventListener.java @@ -0,0 +1,25 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public interface EntityEventListener { + + /** + * Invoked when an entity is inserted. + * + * @param entity The entity that was inserted. + */ + void onEntityInserted(IEntity entity); + + /** + * Invoked when an entity is updated. + * + * @param entity The entity that was updated. + */ + void onEntityUpdated(IEntity entity); + + /** + * Invoked when an entity is removed. + * + * @param entity The entity that was removed. + */ + void onEntityRemoved(IEntity entity); +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java index 1550265e..3af87e67 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java @@ -6,5 +6,6 @@ public enum EntityType { RECOMMENDATION, FALSE_POSITIVE, FALSE_RECOMMENDATION, - DICTIONARY_REMOVAL + DICTIONARY_REMOVAL, + TEMPORARY } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java new file mode 100644 index 00000000..9dc80670 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public class Equality extends Containment { + + public Equality(TextEntity a, TextEntity b) { + + super(a, b); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java index 2e8aba2d..dee5fef9 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java @@ -339,7 +339,9 @@ public interface IEntity { */ default void addMatchedRule(MatchedRule matchedRule) { + boolean wasValid = valid(); getMatchedRuleList().add(matchedRule); + handleStateChange(wasValid); } @@ -353,7 +355,53 @@ public interface IEntity { if (getMatchedRuleList().equals(matchedRules)) { return; } + boolean wasValid = valid(); getMatchedRuleList().addAll(matchedRules); + handleStateChange(wasValid); + } + + + void addEntityEventListener(EntityEventListener listener); + + + void removeEntityEventListener(EntityEventListener listener); + + + default void notifyEntityInserted() { + + for (EntityEventListener listener : getEntityEventListeners()) { + listener.onEntityInserted(this); + } + } + + default void notifyEntityUpdated() { + + for (EntityEventListener listener : getEntityEventListeners()) { + listener.onEntityUpdated(this); + } + } + + default void notifyEntityRemoved() { + + for (EntityEventListener listener : getEntityEventListeners()) { + listener.onEntityRemoved(this); + } + } + + + Collection getEntityEventListeners(); + + + default void handleStateChange(boolean wasValid) { + + if (valid() == wasValid) { + return; + } + if (!removed()) { + notifyEntityUpdated(); + } else { + notifyEntityRemoved(); + } } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java new file mode 100644 index 00000000..f31f2dcd --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public class Intersection extends AbstractRelation { + + public Intersection(TextEntity a, TextEntity b) { + + super(a, b); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java new file mode 100644 index 00000000..5ee4c4dc --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public interface Relation { + + TextEntity getA(); + + + TextEntity getB(); + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index f8334be6..db876644 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -4,6 +4,7 @@ import java.awt.geom.Rectangle2D; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -11,7 +12,10 @@ import java.util.Map; import java.util.PriorityQueue; import java.util.Set; +import org.apache.commons.collections4.map.HashedMap; + import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; @@ -49,7 +53,8 @@ public class TextEntity implements IEntity { @Builder.Default final PriorityQueue matchedRuleList = new PriorityQueue<>(); - final ManualChangeOverwrite manualOverwrite; + @Builder.Default + final ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite(); boolean dictionaryEntry; boolean dossierDictionaryEntry; @@ -68,6 +73,12 @@ public class TextEntity implements IEntity { List intersectingNodes = new LinkedList<>(); SemanticNode deepestFullyContainingNode; + @Builder.Default + Map> relations = new HashMap<>(); + + @Builder.Default + Collection entityEventListeners = new ArrayList<>(); + public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) { @@ -158,12 +169,15 @@ public class TextEntity implements IEntity { public void removeFromGraph() { + remove("FINAL.0.0", "removed completely"); intersectingNodes.forEach(node -> node.getEntities().remove(this)); pages.forEach(page -> page.getEntities().remove(this)); intersectingNodes = new LinkedList<>(); + relations.keySet() + .forEach(entity -> entity.getRelations().remove(this)); + relations = new HashedMap<>(); deepestFullyContainingNode = null; pages = new HashSet<>(); - remove("FINAL.0.0", "removed completely"); } @@ -251,6 +265,20 @@ public class TextEntity implements IEntity { } + public void addManualChange(BaseAnnotation manualChange) { + + manualOverwrite.addChange(manualChange); + notifyEntityUpdated(); + } + + + public void addManualChanges(List manualChanges) { + + manualOverwrite.addChanges(manualChanges); + notifyEntityUpdated(); + } + + public boolean matchesAnnotationId(String manualRedactionId) { return getPositionsOnPagePerPage().stream() @@ -311,4 +339,42 @@ public class TextEntity implements IEntity { .orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value); } + + @Override + public void addEntityEventListener(EntityEventListener listener) { + + entityEventListeners.add(listener); + } + + + @Override + public void removeEntityEventListener(EntityEventListener listener) { + + entityEventListeners.remove(listener); + + } + + + public void computeRelations() { + + for (TextEntity textEntity : this.getDeepestFullyContainingNode().getEntities()) { + if (this.intersects(textEntity) && !this.equals(textEntity) && !textEntity.getEntityType().equals(EntityType.TEMPORARY)) { + if (textEntity.getTextRange().equals(this.getTextRange())) { + textEntity.getRelations().computeIfAbsent(this, k -> new HashSet<>()).add(new Equality(this, textEntity)); + this.getRelations().computeIfAbsent(textEntity, k -> new HashSet<>()).add(new Equality(textEntity, this)); + } else if (textEntity.containedBy(this)) { + textEntity.getRelations().computeIfAbsent(this, k -> new HashSet<>()).add(new Intersection(textEntity, this)); + this.getRelations().computeIfAbsent(textEntity, k -> new HashSet<>()).add(new Containment(this, textEntity)); + } else if (this.containedBy(textEntity)) { + textEntity.getRelations().computeIfAbsent(this, k -> new HashSet<>()).add(new Containment(textEntity, this)); + this.getRelations().computeIfAbsent(textEntity, k -> new HashSet<>()).add(new Intersection(this, textEntity)); + } else { + textEntity.getRelations().computeIfAbsent(this, k -> new HashSet<>()).add(new Intersection(textEntity, this)); + this.getRelations().computeIfAbsent(textEntity, k -> new HashSet<>()).add(new Intersection(this, textEntity)); + } + + } + } + } + } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java index 534534a7..d85d5103 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java @@ -39,7 +39,6 @@ public class Document extends AbstractSemanticNode { @Builder.Default static final SectionIdentifier sectionIdentifier = SectionIdentifier.document(); - @Override public NodeType getType() { diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java index dc8c9276..c65b18b2 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java @@ -1,6 +1,8 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes; import java.awt.geom.Rectangle2D; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Locale; @@ -10,6 +12,7 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.model.document.NodeVisitor; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityEventListener; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; @@ -52,6 +55,9 @@ public class Image extends AbstractSemanticNode implements IEntity { Page page; + @Builder.Default + Collection entityEventListeners = new ArrayList<>(); + @Override public NodeType getType() { @@ -100,6 +106,21 @@ public class Image extends AbstractSemanticNode implements IEntity { } + @Override + public void addEntityEventListener(EntityEventListener listener) { + + entityEventListeners.add(listener); + } + + + @Override + public void removeEntityEventListener(EntityEventListener listener) { + + entityEventListeners.remove(listener); + + } + + @Override public String type() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityCreationUtility.java similarity index 73% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java rename to redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityCreationUtility.java index 59c3fd4c..2e0afd66 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityCreationUtility.java @@ -1,23 +1,27 @@ -package com.iqser.red.service.redaction.v1.server.service.document; +package com.iqser.red.service.redaction.v1.server.utils; import java.util.List; import java.util.Set; +import com.iqser.red.service.redaction.v1.server.model.document.IntersectingNodeVisitor; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import lombok.experimental.UtilityClass; + +@UtilityClass public class EntityCreationUtility { - public static void checkIfBothStartAndEndAreEmpty(String start, String end) { + public void checkIfBothStartAndEndAreEmpty(String start, String end) { checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end)); } - public static void checkIfBothStartAndEndAreEmpty(List start, List end) { + public void checkIfBothStartAndEndAreEmpty(List start, List end) { if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) { throw new IllegalArgumentException("Start and end values are empty!"); @@ -25,7 +29,7 @@ public class EntityCreationUtility { } - public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) { + public int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) { if (textBlock.getNextLinebreak(end) < expandedEnd) { return end; @@ -34,7 +38,7 @@ public class EntityCreationUtility { } - public static Set findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) { + public Set findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) { IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange); @@ -46,7 +50,7 @@ public class EntityCreationUtility { } - public static void addToPages(TextEntity entity) { + public void addToPages(TextEntity entity) { Set pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange()); entity.getPages().addAll(pages); @@ -54,14 +58,14 @@ public class EntityCreationUtility { } - public static void addEntityToNodeEntitySets(TextEntity entity) { + public void addEntityToNodeEntitySets(TextEntity entity) { entity.getIntersectingNodes() .forEach(node -> node.getEntities().add(entity)); } - public static boolean allEntitiesIntersectAndHaveSameTypes(List entitiesToMerge) { + public boolean allEntitiesIntersectAndHaveSameTypes(List entitiesToMerge) { if (entitiesToMerge.isEmpty()) { return true; @@ -79,7 +83,7 @@ public class EntityCreationUtility { } - public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) { + public TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) { if (textBlock.getTextRange().end() == textRange.end()) { return new TextRange(textRange.end(), textRange.end()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityEnrichmentService.java similarity index 68% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java rename to redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityEnrichmentService.java index 4cc70ca9..fbdd30f5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityEnrichmentService.java @@ -1,39 +1,36 @@ -package com.iqser.red.service.redaction.v1.server.service.document; +package com.iqser.red.service.redaction.v1.server.utils; import java.util.Arrays; import java.util.List; import java.util.Objects; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; -import lombok.RequiredArgsConstructor; +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; +import lombok.experimental.UtilityClass; -@Service -@RequiredArgsConstructor +@UtilityClass +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class EntityEnrichmentService { - private final RedactionServiceSettings redactionServiceSettings; + int SURROUNDING_WORDS_OFFSET_WINDOW = 100; + int NUMBER_OF_SURROUNDING_WORDS = 3; public void enrichEntity(TextEntity entity, TextBlock textBlock) { - entity.setValue(textBlock.subSequence(entity.getTextRange()).toString()); entity.setTextAfter(findTextAfter(entity.getTextRange().end(), textBlock)); entity.setTextBefore(findTextBefore(entity.getTextRange().start(), textBlock)); } - private String findTextAfter(int index, TextBlock textBlock) { - - int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().end()); + int endOffset = Math.min(index + SURROUNDING_WORDS_OFFSET_WINDOW, textBlock.getTextRange().end()); String textAfter = textBlock.subSequence(index, endOffset).toString(); if (!textAfter.isBlank()) { List wordsAfter = splitToWordsAndRemoveEmptyWords(textAfter); - int numberOfWordsAfter = Math.min(wordsAfter.size(), redactionServiceSettings.getNumberOfSurroundingWords()); + int numberOfWordsAfter = Math.min(wordsAfter.size(), NUMBER_OF_SURROUNDING_WORDS); if (!wordsAfter.isEmpty()) { return concatWordsAfter(wordsAfter.subList(0, numberOfWordsAfter), textAfter.startsWith(" ")); } @@ -41,14 +38,12 @@ public class EntityEnrichmentService { return ""; } - private String findTextBefore(int index, TextBlock textBlock) { - - int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().start()); + int offsetBefore = Math.max(index - SURROUNDING_WORDS_OFFSET_WINDOW, textBlock.getTextRange().start()); String textBefore = textBlock.subSequence(offsetBefore, index).toString(); if (!textBefore.isBlank()) { List wordsBefore = splitToWordsAndRemoveEmptyWords(textBefore); - int numberOfWordsBefore = Math.min(wordsBefore.size(), redactionServiceSettings.getNumberOfSurroundingWords()); + int numberOfWordsBefore = Math.min(wordsBefore.size(), NUMBER_OF_SURROUNDING_WORDS); if (!wordsBefore.isEmpty()) { return concatWordsBefore(wordsBefore.subList(wordsBefore.size() - numberOfWordsBefore, wordsBefore.size()), textBefore.endsWith(" ")); } @@ -56,36 +51,26 @@ public class EntityEnrichmentService { return ""; } - - private static List splitToWordsAndRemoveEmptyWords(String textAfter) { - - return Arrays.stream(textAfter.split(" ")) + private List splitToWordsAndRemoveEmptyWords(String text) { + return Arrays.stream(text.split(" ")) .filter(word -> !Objects.equals("", word)) .toList(); } - - private static String concatWordsBefore(List words, boolean endWithSpace) { - + private String concatWordsBefore(List words, boolean endWithSpace) { StringBuilder sb = new StringBuilder(); - for (String word : words) { sb.append(word).append(" "); } - String result = sb.toString().trim(); return endWithSpace ? result + " " : result; } - - private static String concatWordsAfter(List words, boolean startWithSpace) { - + private String concatWordsAfter(List words, boolean startWithSpace) { StringBuilder sb = new StringBuilder(); - for (String word : words) { sb.append(word).append(" "); } - String result = sb.toString().trim(); return startWithSpace ? " " + result : result; } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index c9346a17..262655fd 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -12,7 +12,7 @@ plugins { description = "redaction-service-server-v1" -val layoutParserVersion = "0.191.0" +val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" @@ -102,6 +102,10 @@ dependencies { group = "com.iqser.red.service", module = "persistence-service-shared-api-v1" ) + exclude( + group = "com.knecon.fforesight", + module = "document" + ) } testImplementation("com.pdftron:PDFNet:10.11.0") } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java index 7c0e90d0..ac046516 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model; import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.PriorityQueue; @@ -15,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityEventListener; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; @@ -52,7 +54,8 @@ public class PrecursorEntity implements IEntity { @Builder.Default PriorityQueue matchedRuleList = new PriorityQueue<>(); - ManualChangeOverwrite manualOverwrite; + @Builder.Default + ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite(); public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) { @@ -198,6 +201,28 @@ public class PrecursorEntity implements IEntity { } + @Override + public void addEntityEventListener(EntityEventListener listener) { + + throw new UnsupportedOperationException("PrecursorEntity does not support entityEventListeners"); + } + + + @Override + public void removeEntityEventListener(EntityEventListener listener) { + + throw new UnsupportedOperationException("PrecursorEntity does not support entityEventListeners"); + + } + + + @Override + public Collection getEntityEventListeners() { + + throw new UnsupportedOperationException("PrecursorEntity does not support entityEventListeners"); + } + + private static EntityType getEntityType(EntryType entryType) { switch (entryType) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java index 1e4f040e..fee7bad2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java @@ -11,6 +11,7 @@ import org.apache.commons.lang3.StringUtils; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.utils.Patterns; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; @@ -44,6 +45,13 @@ public class Dictionary { } + public boolean containsType(String type) { + + Map levelMap = localAccessMap.get(type); + return !(levelMap == null || levelMap.isEmpty()); + } + + private Level getLevel(boolean isDossierDictionary) { return isDossierDictionary ? Level.DOSSIER : Level.DOSSIER_TEMPLATE; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 1690c19c..29139505 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -7,11 +7,9 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; @@ -25,8 +23,6 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DictionarySearchService { - EntityEnrichmentService entityEnrichmentService; - @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, List semanticNodes) { @@ -38,7 +34,7 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(); dictionary.getDictionarySearch().getBoundaries(node.getTextBlock()) .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary.textRange())) .forEach(match -> { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java index 5e760c87..31174f1b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java @@ -14,7 +14,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.google.common.collect.Sets; @@ -30,7 +29,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import lombok.extern.slf4j.Slf4j; @@ -39,15 +37,7 @@ import lombok.extern.slf4j.Slf4j; @Service public class EntityFindingUtility { - EntityCreationService entityCreationService; - - - @Autowired - public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService) { - - entityCreationService = new EntityCreationService(entityEnrichmentService); - } - + EntityCreationService entityCreationService = new EntityCreationService(); public Optional findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map> entitiesWithSameValue, @@ -191,7 +181,7 @@ public class EntityFindingUtility { return textBlocks.stream() .flatMap(searchImplementation::getBoundaries) - .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet())) + .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, node, Collections.emptySet())) .filter(Optional::isPresent) .map(Optional::get) .distinct() @@ -218,7 +208,7 @@ public class EntityFindingUtility { return textBlocks.stream() .flatMap(tb -> searchImplementation.getBoundaries(tb) .filter(textRange -> entityCreationService.isValidEntityTextRange(tb, textRange))) - .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, document, Collections.emptySet())) + .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, document, Collections.emptySet())) .filter(Optional::isPresent) .map(Optional::get) .distinct() @@ -232,7 +222,7 @@ public class EntityFindingUtility { return searchImplementation.getBoundaries(document.getTextBlock()) .filter(textRange -> entityCreationService.isValidEntityTextRange(document.getTextBlock(), textRange)) - .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, document, Collections.emptySet())) + .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.TEMPORARY, document, Collections.emptySet())) .filter(Optional::isPresent) .map(Optional::get) .distinct() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index ba644071..c92d0d42 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -431,12 +431,11 @@ public class EntityLogCreatorService { private static EntryType getEntryType(EntityType entityType) { return switch (entityType) { - case ENTITY -> EntryType.ENTITY; + case ENTITY, TEMPORARY -> EntryType.ENTITY; case HINT -> EntryType.HINT; - case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; + case FALSE_POSITIVE, DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE; case RECOMMENDATION -> EntryType.RECOMMENDATION; case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; - case DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE; }; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index 42f10e75..7b9b3332 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.service; import java.awt.geom.Rectangle2D; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -17,6 +18,8 @@ import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; @@ -44,18 +47,19 @@ public class ManualChangesApplicationService { public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) { entityToBeReCategorized.getMatchedRuleList().clear(); - entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization); if (manualRecategorization.getType() == null) { return; } if (entityToBeReCategorized instanceof Image image) { + entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization); image.setImageType(ImageType.fromString(manualRecategorization.getType())); return; } if (entityToBeReCategorized instanceof TextEntity textEntity) { + textEntity.addManualChange(manualRecategorization); textEntity.setType(manualRecategorization.getType()); } } @@ -76,6 +80,8 @@ public class ManualChangesApplicationService { @Deprecated public void resizeEntityAndReinsert(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) { + entityToBeResized.notifyEntityRemoved(); + PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage() .stream() .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId())) @@ -103,7 +109,7 @@ public class ManualChangesApplicationService { .stream() .flatMap(Collection::stream) .forEach(TextEntity::removeFromGraph); - return; + break; } possibleEntities.values() @@ -114,9 +120,15 @@ public class ManualChangesApplicationService { if (node.hasParent()) { node = node.getParent(); } else { - break; + node = null; } } + + entityToBeResized.getRelations().keySet() + .forEach(textEntity -> textEntity.getRelations().remove(entityToBeResized)); + entityToBeResized.setRelations(new HashMap<>()); + entityToBeResized.computeRelations(); + entityToBeResized.notifyEntityInserted(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index 934f9a9a..6fecda2b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -1,17 +1,9 @@ package com.iqser.red.service.redaction.v1.server.service.document; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addEntityToNodeEntitySets; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.addToPages; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.allEntitiesIntersectAndHaveSameTypes; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.checkIfBothStartAndEndAreEmpty; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.findIntersectingSubNodes; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.toLineAfterTextRange; -import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween; import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; import java.util.List; @@ -21,7 +13,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; -import org.kie.api.runtime.KieSession; import com.google.common.base.Functions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; @@ -38,35 +29,26 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.service.drools.KieSessionUpdater; +import com.iqser.red.service.redaction.v1.server.utils.EntityCreationUtility; +import com.iqser.red.service.redaction.v1.server.utils.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; -import lombok.RequiredArgsConstructor; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j -@RequiredArgsConstructor +@NoArgsConstructor public class EntityCreationService { - private final EntityEnrichmentService entityEnrichmentService; - private final KieSession kieSession; - private final Set nodesInKieSession; // empty set means all nodes are in kieSession + KieSessionUpdater kieSessionUpdater; - public EntityCreationService(EntityEnrichmentService entityEnrichmentService) { + public EntityCreationService(KieSessionUpdater kieSessionUpdater) { - this.entityEnrichmentService = entityEnrichmentService; - this.kieSession = null; - this.nodesInKieSession = Collections.emptySet(); - } - - - public EntityCreationService(EntityEnrichmentService entityEnrichmentService, KieSession kieSession) { - - this.entityEnrichmentService = entityEnrichmentService; - this.kieSession = kieSession; - this.nodesInKieSession = Collections.emptySet(); + this.kieSessionUpdater = kieSessionUpdater; } @@ -83,7 +65,7 @@ public class EntityCreationService { */ public Stream betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(start, stop); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(start, stop); List startTextRanges = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock()); List stopTextRanges = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock()); @@ -105,7 +87,7 @@ public class EntityCreationService { */ public Stream betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(start, stop); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(start, stop); List startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock()); List stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock()); @@ -127,7 +109,7 @@ public class EntityCreationService { */ public Stream betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(start, stop); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(start, stop); List startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock()); List stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock()); @@ -154,7 +136,7 @@ public class EntityCreationService { */ public Stream betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(start, stop); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(start, stop); List startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock()); List stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock()); @@ -181,7 +163,7 @@ public class EntityCreationService { */ public Stream betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(start, stop); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(start, stop); List startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock()); List stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock()); @@ -208,7 +190,7 @@ public class EntityCreationService { */ public Stream betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(start, stop); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(start, stop); List startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock()); List stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock()); @@ -235,7 +217,7 @@ public class EntityCreationService { */ public Stream betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(start, stop); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(start, stop); List startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock()); List stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock()); @@ -266,7 +248,7 @@ public class EntityCreationService { */ public Stream betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(start, stop); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(start, stop); List startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock()); List stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock()); @@ -297,7 +279,7 @@ public class EntityCreationService { */ public Stream shortestBetweenAnyString(List starts, List stops, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(starts, stops); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(starts, stops); List startTextRanges = RedactionSearchUtility.findTextRangesByList(starts, node.getTextBlock()); List stopTextRanges = RedactionSearchUtility.findTextRangesByList(stops, node.getTextBlock()); @@ -319,7 +301,7 @@ public class EntityCreationService { */ public Stream shortestBetweenAnyStringIgnoreCase(List starts, List stops, String type, EntityType entityType, SemanticNode node) { - checkIfBothStartAndEndAreEmpty(starts, stops); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(starts, stops); List startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock()); List stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock()); @@ -342,7 +324,7 @@ public class EntityCreationService { */ public Stream shortestBetweenAnyStringIgnoreCase(List starts, List stops, String type, EntityType entityType, SemanticNode node, int limit) { - checkIfBothStartAndEndAreEmpty(starts, stops); + EntityCreationUtility.checkIfBothStartAndEndAreEmpty(starts, stops); List startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock()); List stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock()); @@ -498,7 +480,7 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); SearchImplementation searchImplementation = new SearchImplementation(strings, false); return searchImplementation.getBoundaries(textBlock) - .map(boundary -> toLineAfterTextRange(textBlock, boundary)) + .map(boundary -> EntityCreationUtility.toLineAfterTextRange(textBlock, boundary)) .filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -520,7 +502,7 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); SearchImplementation searchImplementation = new SearchImplementation(strings, true); return searchImplementation.getBoundaries(textBlock) - .map(boundary -> toLineAfterTextRange(textBlock, boundary)) + .map(boundary -> EntityCreationUtility.toLineAfterTextRange(textBlock, boundary)) .filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -542,7 +524,7 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); return RedactionSearchUtility.findTextRangesByString(string, textBlock) .stream() - .map(boundary -> toLineAfterTextRange(textBlock, boundary)) + .map(boundary -> EntityCreationUtility.toLineAfterTextRange(textBlock, boundary)) .filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -564,7 +546,7 @@ public class EntityCreationService { TextBlock textBlock = node.getTextBlock(); return RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, textBlock) .stream() - .map(boundary -> toLineAfterTextRange(textBlock, boundary)) + .map(boundary -> EntityCreationUtility.toLineAfterTextRange(textBlock, boundary)) .filter(boundary -> isValidEntityTextRange(textBlock, boundary)) .map(boundary -> byTextRange(boundary, type, entityType, node)) .filter(Optional::isPresent) @@ -949,7 +931,7 @@ public class EntityCreationService { public Optional bySuffixExpansionRegex(TextEntity entity, String regexPattern) { int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern); - expandedEnd = truncateEndIfLineBreakIsBetween(entity.getTextRange().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock()); + expandedEnd = EntityCreationUtility.truncateEndIfLineBreakIsBetween(entity.getTextRange().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock()); return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode()); } @@ -1012,12 +994,12 @@ public class EntityCreationService { } return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there } - boolean added = addEntityToGraph(entity, node.getDocumentTree()); - if (!added) { - return Optional.empty(); - } + + addListenerToEntity(entity); + node.getDocumentTree().addEntityToGraph(entity); + entity.addEngines(engines); - insertToKieSession(entity); + return Optional.of(entity); } @@ -1057,7 +1039,7 @@ public class EntityCreationService { @Deprecated(forRemoval = true) public TextEntity mergeEntitiesOfSameType(List entitiesToMerge, String type, EntityType entityType, SemanticNode node) { - if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) { + if (!EntityCreationUtility.allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) { throw new IllegalArgumentException("Provided entities can not be merged, since they do not intersect or are not the same type!" + entitiesToMerge); } if (entitiesToMerge.isEmpty()) { @@ -1082,17 +1064,16 @@ public class EntityCreationService { .map(TextEntity::getManualOverwrite) .map(ManualChangeOverwrite::getManualChangeLog) .flatMap(Collection::stream) - .forEach(manualChange -> mergedEntity.getManualOverwrite().addChange(manualChange)); + .forEach(mergedEntity::addManualChange); mergedEntity.setDictionaryEntry(entitiesToMerge.stream() .anyMatch(TextEntity::isDictionaryEntry)); mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream() .anyMatch(TextEntity::isDossierDictionaryEntry)); - entityEnrichmentService.enrichEntity(mergedEntity, node.getTextBlock()); + EntityEnrichmentService.enrichEntity(mergedEntity, node.getTextBlock()); addEntityToGraph(mergedEntity, node); - insertToKieSession(mergedEntity); entitiesToMerge.stream() .filter(e -> !e.equals(mergedEntity)) @@ -1147,26 +1128,13 @@ public class EntityCreationService { TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException( "No entity present!")); - newEntity.getManualOverwrite().addChanges(entity.getManualOverwrite().getManualChangeLog()); + newEntity.addManualChanges(entity.getManualOverwrite().getManualChangeLog()); newEntity.setDictionaryEntry(entity.isDictionaryEntry()); newEntity.setDossierDictionaryEntry(entity.isDossierDictionaryEntry()); return newEntity; } - /** - * Inserts a text entity into the kieSession for further processing. - * - * @param textEntity The merged text entity to insert. - */ - public void insertToKieSession(TextEntity textEntity) { - - if (kieSession != null) { - kieSession.insert(textEntity); - } - } - - /** * Creates a text entity based on a Named Entity Recognition (NER) entity. * @@ -1454,7 +1422,8 @@ public class EntityCreationService { .ifPresent(e -> addDuplicateEntityToGraph(e, entity.getTextRange(), node)); } else { - addEntityToGraph(entity, documentTree); + addListenerToEntity(entity); + documentTree.addEntityToGraph(entity); } } @@ -1492,7 +1461,7 @@ public class EntityCreationService { entityToDuplicate.setDeepestFullyContainingNode(deepestSharedNode); - Set additionalIntersectingNodes = findIntersectingSubNodes(deepestSharedNode, newTextRange); + Set additionalIntersectingNodes = EntityCreationUtility.findIntersectingSubNodes(deepestSharedNode, newTextRange); additionalIntersectingNodes.forEach(additionalIntersectingNode -> { if (entityToDuplicate.getIntersectingNodes().contains(additionalIntersectingNode)) { @@ -1506,23 +1475,13 @@ public class EntityCreationService { } - private boolean addEntityToGraph(TextEntity entity, DocumentTree documentTree) { + private void addListenerToEntity(TextEntity textEntity) { - documentTree.getRoot().getNode().addThisToEntityIfIntersects(entity); - - if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes() - .stream() - .anyMatch(node -> !nodesInKieSession.contains(node))) { - entity.removeFromGraph(); - return false; + if(kieSessionUpdater != null) { + textEntity.addEntityEventListener(kieSessionUpdater); } - - TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock(); - entityEnrichmentService.enrichEntity(entity, textBlock); - - addToPages(entity); - addEntityToNodeEntitySets(entity); - return true; } + + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index bbad6457..32b7e8be 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.service.drools; import static com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService.RULES_LOGGER_GLOBAL; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -29,11 +28,12 @@ import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; import com.iqser.red.service.redaction.v1.server.logger.TrackingAgendaEventListener; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.websocket.WebSocketService; import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException; @@ -51,7 +51,6 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class EntityDroolsExecutionService { - EntityEnrichmentService entityEnrichmentService; ObservationRegistry observationRegistry; ManualChangesApplicationService manualChangesApplicationService; RedactionServiceSettings settings; @@ -95,81 +94,86 @@ public class EntityDroolsExecutionService { KieSession kieSession = kieContainer.newKieSession(); - Set nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes() - .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document); - - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession, nodesInKieSession); - RulesLogger logger = new RulesLogger(webSocketService, context); - if (settings.isDroolsDebug()) { - logger.enableAgendaTracking(); - logger.enableObjectTracking(); - } - kieSession.addEventListener(new TrackingAgendaEventListener(logger)); - kieSession.addEventListener(new ObjectTrackingEventListener(logger)); - - kieSession.setGlobal("document", document); - kieSession.setGlobal("entityCreationService", entityCreationService); - kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); - kieSession.setGlobal("dictionary", dictionary); - - if (hasGlobalWithName(kieSession, RULES_LOGGER_GLOBAL)) { - kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger); - } - - kieSession.insert(document); - - document.getEntities() - .forEach(kieSession::insert); - - sectionsToAnalyze.forEach(kieSession::insert); - - sectionsToAnalyze.stream() - .flatMap(SemanticNode::streamAllSubNodes) - .forEach(kieSession::insert); - - document.getPages() - .forEach(kieSession::insert); - - fileAttributes.stream() - .filter(f -> f.getValue() != null) - .forEach(kieSession::insert); - - if (manualRedactions != null) { - manualRedactions.buildAll() - .stream() - .filter(BaseAnnotation::isLocal) - .forEach(kieSession::insert); - } - - kieSession.insert(nerEntities); - - kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); - - CompletableFuture completableFuture = CompletableFuture.supplyAsync(() -> { - kieSession.fireAllRules(); - return null; - }); - try { - completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS); - } catch (ExecutionException e) { - logger.error(e, "Exception during rule execution"); - kieSession.dispose(); - if (e.getCause() instanceof TimeoutException) { - throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.ENTITY); + KieSessionUpdater kieSessionUpdater = new KieSessionUpdater(kieSession); + EntityCreationService entityCreationService = new EntityCreationService(kieSessionUpdater); + RulesLogger logger = new RulesLogger(webSocketService, context); + if (settings.isDroolsDebug()) { + logger.enableAgendaTracking(); + logger.enableObjectTracking(); } - throw new RuntimeException(e); - } catch (InterruptedException e) { - logger.error(e, "Exception during rule execution"); - kieSession.dispose(); - throw new RuntimeException(e); - } catch (TimeoutException e) { - throw new DroolsTimeoutException(String.format("The file %s caused a timeout",context.getFileId()), e, false, RuleFileType.ENTITY); - } + kieSession.addEventListener(new TrackingAgendaEventListener(logger)); + kieSession.addEventListener(new ObjectTrackingEventListener(logger)); - List resultingFileAttributes = getFileAttributes(kieSession); - kieSession.dispose(); - return resultingFileAttributes; + kieSession.setGlobal("document", document); + kieSession.setGlobal("entityCreationService", entityCreationService); + kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); + kieSession.setGlobal("dictionary", dictionary); + + if (hasGlobalWithName(kieSession, RULES_LOGGER_GLOBAL)) { + kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger); + } + + kieSession.insert(document); + sectionsToAnalyze.forEach(kieSession::insert); + + sectionsToAnalyze.stream() + .flatMap(SemanticNode::streamAllSubNodes) + .forEach(semanticNode -> { + if (semanticNode instanceof Image image) { + image.addEntityEventListener(kieSessionUpdater); + image.notifyEntityInserted(); + } else { + kieSession.insert(semanticNode); + } + }); + + for (TextEntity textEntity : document.getEntities()) { + textEntity.addEntityEventListener(kieSessionUpdater); + textEntity.notifyEntityInserted(); + } + + document.getPages() + .forEach(kieSession::insert); + + fileAttributes.stream() + .filter(f -> f.getValue() != null) + .forEach(kieSession::insert); + + if (manualRedactions != null) { + manualRedactions.buildAll() + .stream() + .filter(BaseAnnotation::isLocal) + .forEach(kieSession::insert); + } + + kieSession.insert(nerEntities); + + kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); + + CompletableFuture completableFuture = CompletableFuture.runAsync(kieSession::fireAllRules); + + try { + completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS); + } catch (ExecutionException e) { + logger.error(e, "Exception during rule execution"); + if (e.getCause() instanceof TimeoutException) { + throw new DroolsTimeoutException(String.format("The file %s caused a timeout", context.getFileId()), e, false, RuleFileType.ENTITY); + } + throw new RuntimeException(e); + } catch (InterruptedException e) { + logger.error(e, "Exception during rule execution"); + throw new RuntimeException(e); + } catch (TimeoutException e) { + throw new DroolsTimeoutException(String.format("The file %s caused a timeout", context.getFileId()), e, false, RuleFileType.ENTITY); + } + + List resultingFileAttributes = getFileAttributes(kieSession); + return resultingFileAttributes; + + } finally { + kieSession.dispose(); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java new file mode 100644 index 00000000..a540bf71 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java @@ -0,0 +1,98 @@ +package com.iqser.red.service.redaction.v1.server.service.drools; + +import java.util.Collection; +import java.util.Collections; +import java.util.function.Consumer; + +import org.kie.api.runtime.KieSession; +import org.kie.api.runtime.rule.FactHandle; + +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityEventListener; +import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class KieSessionUpdater implements EntityEventListener { + + KieSession kieSession; + + + @Override + public void onEntityInserted(IEntity entity) { + + handleOnEntityEvent(entity, kieSession::insert); + kieSession.insert(entity); + } + + + @Override + public void onEntityUpdated(IEntity entity) { + + handleOnEntityEvent(entity, this::updateFactIfPresent); + kieSession.update(kieSession.getFactHandle(entity), entity); + } + + + @Override + public void onEntityRemoved(IEntity entity) { + + handleOnEntityEvent(entity, this::deleteFactIfPresent); + kieSession.delete(kieSession.getFactHandle(entity)); + } + + + private void handleOnEntityEvent(IEntity entity, Consumer consumer) { + + if (entity instanceof TextEntity textEntity) { + updateIntersectingNodes(textEntity); + textEntity.getRelations().values() + .stream() + .flatMap(Collection::stream) + .forEach(consumer); + textEntity.getRelations().keySet() + .forEach(k -> k.getRelations().getOrDefault(textEntity, Collections.emptySet()) + .forEach(consumer)); + } + + if (entity instanceof Image image) { + SemanticNode parent = image; + while (parent.hasParent()) { + parent = parent.getParent(); + kieSession.update(kieSession.getFactHandle(parent), parent); + } + } + } + + + private void updateIntersectingNodes(TextEntity textEntity) { + + textEntity.getIntersectingNodes() + .forEach(this::updateFactIfPresent); + } + + + private void updateFactIfPresent(Object o) { + + FactHandle factHandle = kieSession.getFactHandle(o); + if (factHandle != null) { + kieSession.update(factHandle, o); + } + } + + + private void deleteFactIfPresent(Object o) { + + FactHandle factHandle = kieSession.getFactHandle(o); + if (factHandle != null) { + kieSession.delete(factHandle); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl index b1c04d70..e9cf3990 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl @@ -13,12 +13,14 @@ import java.util.stream.Stream; import java.util.Optional; import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; +import com.iqser.red.service.redaction.v1.server.model.document.*; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; @@ -47,8 +49,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; -import com.iqser.red.service.redaction.v1.server.data.LayoutEngineProto.LayoutEngine; - global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -1311,8 +1311,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1324,8 +1322,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -1336,10 +1332,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1349,9 +1343,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -1362,9 +1354,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -1375,8 +1365,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -1389,9 +1377,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1402,7 +1388,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1414,8 +1400,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -1436,7 +1420,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -1446,8 +1429,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -1458,71 +1440,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1530,22 +1555,33 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.7 @@ -1563,22 +1599,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1607,11 +1654,16 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end @@ -1624,7 +1676,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl index 489bad7b..e8152dd3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl @@ -18,8 +18,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml index 02d4f3ec..3233828b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml @@ -14,4 +14,5 @@ + \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index 5d43c3f2..d1b66a84 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.GZIPInputStream; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -61,23 +62,30 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryFactory; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver; import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer; +import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver; import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor; import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; +import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantProvider; import com.knecon.fforesight.tenantcommons.model.MongoDBConnection; +import com.pdftron.pdf.PDFNet; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -114,7 +122,7 @@ import lombok.extern.slf4j.Slf4j; FileType.DOCUMENT_POSITION, FileType.DOCUMENT_STRUCTURE, FileType.DOCUMENT_TEXT); - Path dossierTemplateToUse = Path.of("/home/kschuettler/Downloads/New Folder/DOSSIER_TEMPLATE"); // Add your dossier-template here + Path dossierTemplateToUse = Path.of("/home/kschuettler/iqser/business-logic/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here ObjectMapper mapper = ObjectMapperFactory.create(); final String TENANT_ID = "tenant"; TestDossierTemplate testDossierTemplate; @@ -151,13 +159,26 @@ import lombok.extern.slf4j.Slf4j; private TenantProvider tenantProvider; @Autowired private DictionaryFactory dictionaryFactory; + @Autowired + private LayoutParsingPipeline layoutParsingPipeline; + @MockBean + private RedactionMessageReceiver redactionMessageReceiver; + + + @BeforeAll + public static void init() { + + synchronized (PDFNet.class) { + PDFNet.initialize("demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a"); + } + } @Test @SneakyThrows public void runAnalysisEnd2End() { - String folder = "/home/kschuettler/Downloads/New Folder/436e4a2a-0ba3-4d3c-9944-c355f5c1cca2"; // Should contain all files from minio directly, still zipped. Can contain multiple files. + String folder = "/home/kschuettler/Dokumente/analysisend2end/file0"; // Should contain all files from minio directly, still zipped. Can contain multiple files. Path absoluteFolderPath; if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path @@ -171,8 +192,18 @@ import lombok.extern.slf4j.Slf4j; List analyzeRequests = prepareStorageForFolder(absoluteFolderPath); log.info("Found {} distinct fileIds with all required files", analyzeRequests.size()); for (int i = 0; i < analyzeRequests.size(); i++) { - long start = System.currentTimeMillis(); + AnalyzeRequest analyzeRequest = analyzeRequests.get(i); + Path nerEntitiesFile = absoluteFolderPath.resolve(analyzeRequest.getFileId() + ".NER_ENTITIES.json"); + if (!Files.exists(nerEntitiesFile)) { + storageService.storeJSONObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + FileType.NER_ENTITIES), + new NerEntities()); + + } + long start = System.currentTimeMillis(); log.info("----------------------------------------------------------------------------------"); log.info("{}/{}: Starting analysis for file {}", i + 1, analyzeRequests.size(), analyzeRequest.getFileId()); analyzeService.analyze(analyzeRequest); @@ -192,7 +223,7 @@ import lombok.extern.slf4j.Slf4j; manualRedactionEntry.setValue("7232"); manualRedactionEntry.setReason( "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)"); - manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + manualRedactionEntry.setLegalBasis("personal_data_geolocation_article_39e3"); manualRedactionEntry.setProcessedDate(OffsetDateTime.now()); manualRedactionEntry.setRequestDate(OffsetDateTime.now()); manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build())); @@ -303,7 +334,7 @@ import lombok.extern.slf4j.Slf4j; AnalyzeRequest request = new AnalyzeRequest(); request.setDossierId(UUID.randomUUID().toString()); - request.setFileId(UUID.randomUUID().toString()); + request.setFileId(fileName); request.setDossierTemplateId(testDossierTemplate.id); request.setAnalysisNumber(-1); @@ -320,18 +351,72 @@ import lombok.extern.slf4j.Slf4j; Set missingFileTypes = Sets.difference(REQUIRED_FILES, uploadedFileTypes); - if (!missingFileTypes.isEmpty()) { - log.error("Folder {} is missing files of type {}", - folder.toFile(), - missingFileTypes.stream() - .map(Enum::toString) - .collect(Collectors.joining(", "))); - return Optional.empty(); + if (!missingFileTypes.isEmpty() && !missingFileTypes.contains(FileType.ORIGIN)) { + runLayoutParsingAndSaveFilesToFolder(folder, uploadedFileTypes, request); } + +// if (!missingFileTypes.isEmpty()) { +// log.error("Folder {} is missing files of type {}", +// folder.toFile(), +// missingFileTypes.stream() +// .map(Enum::toString) +// .collect(Collectors.joining(", "))); +// return Optional.empty(); +// } return Optional.of(request); } + private void runLayoutParsingAndSaveFilesToFolder(Path folder, Set uploadedFileTypes, AnalyzeRequest request) throws IOException { + + uploadImageAndTableFilesIfMissing(uploadedFileTypes, request); + + LayoutParsingRequest layoutParsingRequest = LayoutParsingRequestProvider.build(LayoutParsingType.DOCUMINE_OLD, request); + layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest); + + try { + storeFileFromStorage(TENANT_ID, layoutParsingRequest.structureFileStorageId(), folder); + storeFileFromStorage(TENANT_ID, layoutParsingRequest.textBlockFileStorageId(), folder); + storeFileFromStorage(TENANT_ID, layoutParsingRequest.positionBlockFileStorageId(), folder); + storeFileFromStorage(TENANT_ID, layoutParsingRequest.pageFileStorageId(), folder); + } catch (IOException e) { + log.error("Failed to store files from storage to folder {}", folder, e); + } + } + + + private void uploadImageAndTableFilesIfMissing(Set uploadedFileTypes, AnalyzeRequest request) throws IOException { + + if (!uploadedFileTypes.contains(FileType.TABLES)) { + var cvServiceResponse = "files/cv_service_empty_response.json"; + ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponse); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES), + cvServiceResponseFileResource.getInputStream()); + } + if (!uploadedFileTypes.contains(FileType.IMAGE_INFO)) { + var imageServiceResponse = "files/empty_image_response.json"; + ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponse); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO), + imageServiceResponseFileResource.getInputStream()); + } + } + + + private void storeFileFromStorage(String tenantId, String storageId, Path folder) throws IOException { + + var inputStream = storageService.getObject(tenantId, storageId); + try (FileOutputStream fileOut = new FileOutputStream(folder.toString() + "/" + storageId.split("/")[1])) { + fileOut.write(inputStream.getContentAsByteArray()); + } catch (IOException e) { + e.printStackTrace(); + } + log.info("Stored file {} to {}", storageId, folder); + + } + + private static Stream findFilesToUpload(String fileName, Path folder, Set endingsToUpload) throws IOException { return Files.walk(folder) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index af47a272..3f49512d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1262,7 +1262,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { .value("0049 331 441 551 14") .requestDate(OffsetDateTime.now()) .fileId(TEST_FILE_ID) - .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") + .legalBasis("vertebrate_study_personal_data_geolocation_article_39e2") .user("user") .build())) .build()); @@ -1315,7 +1315,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { .value("0049 331 441 551 14") .requestDate(OffsetDateTime.now()) .fileId(TEST_FILE_ID) - .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") + .legalBasis("vertebrate_study_personal_data_geolocation_article_39e2") .user("user") .build())) .recategorizations(Set.of(ManualRecategorization.builder() @@ -1654,35 +1654,46 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { .get(); request.setManualRedactions(ManualRedactions.builder() + .entriesToAdd(Set.of(ManualRedactionEntry.builder() + .annotationId("newId") + .fileId(TEST_FILE_ID) + .user("user") + .requestDate(OffsetDateTime.now()) + .value("David Ksenia") + .type("CBI_author") + .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(295.2f).width(65.59f).height(12.64f).page(1).build())) + .build())) .resizeRedactions(Set.of(ManualResizeRedaction.builder() .updateDictionary(false) - .annotationId(davidKsenia.getId()) + .annotationId("newId") .fileId(TEST_FILE_ID) .user("user") .requestDate(OffsetDateTime.now()) .value("David") - .positions(List.of(Rectangle.builder() - .topLeftX(56.8f) - .topLeftY(293.564f) - .width(29.2922f) - .height(15.408f) - .page(1) - .build())) + .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(29.2922f).height(15.408f).page(1).build())) .addToAllDossiers(false) .build())) .build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var resizedEntity = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getId().equals("newId")) + .findFirst() + .get(); + + var removedEntity = entityLog.getEntityLogEntry() .stream() .filter(e -> e.getId().equals(davidKsenia.getId())) .findFirst() .get(); - assertEquals(resizedEntity.getState(), EntryState.APPLIED); - assertEquals(resizedEntity.getValue(), "David"); - assertEquals(1, resizedEntity.getManualChanges().size()); - assertEquals(resizedEntity.getManualChanges() - .get(0).getManualRedactionType(), ManualRedactionType.RESIZE); + + assertEquals(EntryState.APPLIED, resizedEntity.getState()); + assertEquals("David", resizedEntity.getValue()); + assertEquals(2, resizedEntity.getManualChanges().size()); + assertEquals(1, resizedEntity.getEngines().size()); + + assertEquals(EntryState.REMOVED, removedEntity.getState()); } @@ -1915,7 +1926,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { .reason("(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)") .addToDossierDictionary(false) .addToDictionary(false) - .legalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002") + .legalBasis("personal_data_geolocation_article_39e3") .rectangle(false) .positions(List.of(Rectangle.builder() .topLeftX(270.844f) @@ -2262,14 +2273,14 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { manualRedactionEntry.setValue(valueToAdd); manualRedactionEntry.setReason("Author found, removed by manual override"); manualRedactionEntry.setSection("Header: This is my test"); - manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + manualRedactionEntry.setLegalBasis("personal_data_geolocation_article_39e3"); manualRedactionEntry.setTextBefore("Lorem My Ipsum "); manualRedactionEntry.setTextAfter("Crandu Seku Laku"); manualRedactionEntry.setPositions(localfullPositions); ManualForceRedaction forceRequest = new ManualForceRedaction(); forceRequest.setAnnotationId(manualAddId); - forceRequest.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + forceRequest.setLegalBasis("personal_data_geolocation_article_39e3"); forceRequest.setUser("test"); forceRequest.setRequestDate(OffsetDateTime.now()); @@ -2353,8 +2364,10 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { assertEquals(entityLog.getEntityLogEntry().size(), 3); } + @Test public void testPurityRule() { + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); @@ -2363,7 +2376,10 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { analyzeService.analyze(request); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var entriesCount = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().toLowerCase(Locale.ENGLISH).startsWith("purity")).collect(Collectors.toList()).size(); + var entriesCount = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getValue().toLowerCase(Locale.ENGLISH).startsWith("purity")) + .collect(Collectors.toList()).size(); assertEquals(7, entriesCount); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java index 2fcce7ce..df7d6e50 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java @@ -14,7 +14,6 @@ import org.junit.jupiter.api.Test; import org.kie.api.runtime.KieSession; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; @@ -30,12 +29,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.drools.KieSessionUpdater; public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; @Mock @@ -46,7 +43,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr public void createEntityCreationService() { MockitoAnnotations.initMocks(this); - entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); + entityCreationService = new EntityCreationService(new KieSessionUpdater(kieSession)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index a0de089d..78442fb6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -39,7 +39,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService; import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -53,8 +52,6 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest { @Autowired private DictionaryService dictionaryService; - @Autowired - private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; @Autowired @@ -68,7 +65,7 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest { @BeforeEach public void stubClients() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(); TenantContext.setTenantId("redaction"); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java index a98865ae..43e7f9f1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java @@ -5,28 +5,22 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.List; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; public class SearchImplementationTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; - - @Test public void testSearchImplementationWithPunctuation() { Document document = buildGraph("files/Minimal Examples/TestPunctuation"); SearchImplementation searchImplementation = new SearchImplementation(List.of("Kuhn, J. O."), true); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(); List entities = entityCreationService.bySearchImplementation(searchImplementation, "CBI_author", EntityType.ENTITY, document) .toList(); assertEquals(2, entities.size()); @@ -38,7 +32,7 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest { Document document = buildGraph("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections"); SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(); List entities = entityCreationService.bySearchImplementation(searchImplementation, "dossier_redaction", EntityType.ENTITY, document) .toList(); assertEquals(2, entities.size()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java index e2e33390..c2e76757 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; @@ -21,7 +20,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility; import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService; @@ -32,10 +30,6 @@ import lombok.SneakyThrows; public class TableTest extends BuildDocumentIntegrationTest { private static final boolean DRAW_FILE = false; - - @Autowired - private EntityEnrichmentService entityEnrichmentService; - private EntityCreationService entityCreationService; private static final String TYPE_1 = "type1"; @@ -52,7 +46,7 @@ public class TableTest extends BuildDocumentIntegrationTest { @BeforeEach public void createTable() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(); String fileName = "files/Minimal Examples/BasicTable.pdf"; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java index b8fd95fe..4db674a4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java @@ -23,7 +23,6 @@ import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService; import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.service.drools.RuleFileParser; @@ -36,8 +35,6 @@ class DroolsValidationServiceTest { @MockBean RulesClient rulesClient; @MockBean - EntityEnrichmentService entityEnrichmentService; - @MockBean RedactionServiceSettings redactionServiceSettings; @@ -455,7 +452,7 @@ class DroolsValidationServiceTest { $entity.redact( "CBI.1.0", "Author found", - "Article 39(e)(3) of Regulation (EC) No 178/2002" + "personal_data_geolocation_article_39e3" ); end """; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 6ea85b4c..bc549df1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -23,7 +23,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -54,7 +53,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityTyp import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -67,8 +65,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl"); private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl"); - @Autowired - private EntityEnrichmentService entityEnrichmentService; + private EntityCreationService entityCreationService; @@ -76,7 +73,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { @BeforeEach public void createServices() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index b370bf1e..b5160d8a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -20,12 +20,9 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityTyp import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; @@ -33,7 +30,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { @BeforeEach public void createServices() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java index 212f5ae9..9a79a0ee 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java @@ -35,15 +35,12 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService; import lombok.SneakyThrows; public class PrecursorEntityTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; @Autowired private EntityFromPrecursorCreationService entityFromPrecursorCreationService; @@ -104,7 +101,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { public void createFoundManualRedaction2() { Document document = buildGraph("files/Minimal Examples/TestPunctuation"); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(); List tempEntities = entityCreationService.byString("Kuhn, J. O.", "CBI_author", EntityType.ENTITY, document) .toList(); @@ -159,9 +156,9 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { private DocumentAndEntity createFoundManualRedaction() { Document document = buildGraph("files/syngenta/CustomerFiles/VV-919901.pdf"); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(); - List tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document) + List tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.TEMPORARY, document) .toList(); assertFalse(tempEntities.isEmpty()); var tempEntity = tempEntities.get(0); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index 3c919839..e3592853 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -19,7 +19,6 @@ import org.apache.pdfbox.Loader; import org.apache.pdfbox.pdmodel.PDDocument; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import com.iqser.red.commons.jackson.ObjectMapperFactory; @@ -32,7 +31,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionO import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter; import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility; @@ -42,8 +40,6 @@ import lombok.extern.slf4j.Slf4j; @Slf4j class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { - @Autowired - private EntityEnrichmentService entityEnrichmentService; private EntityCreationService entityCreationService; @@ -58,7 +54,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { @BeforeEach public void createEntityCreationService() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java index 06cfb455..94cfd1ec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java @@ -25,7 +25,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; +import com.iqser.red.service.redaction.v1.server.service.drools.KieSessionUpdater; @ExtendWith(MockitoExtension.class) @Import(RulesIntegrationTest.TestConfiguration.class) @@ -33,8 +33,6 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { protected static final String RULES = "drools/rules.drl"; - @Autowired - protected EntityEnrichmentService entityEnrichmentService; @Autowired protected ManualChangesApplicationService manualChangesApplicationService; protected EntityCreationService entityCreationService; @@ -82,7 +80,7 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { Dictionary dict = Mockito.mock(Dictionary.class); kieSession = kieContainer.newKieSession(); - entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession); + entityCreationService = new EntityCreationService(new KieSessionUpdater(kieSession)); kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); kieSession.setGlobal("entityCreationService", entityCreationService); kieSession.setGlobal("dictionary", dict); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt index cbcbfe3d..86c5d8ca 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt @@ -11,7 +11,6 @@ Sude Halide Nurullah Xinyi Y. Tao Dorn Prasher -David annotation J.B. RASCLE （果梗を除去したもの） diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/most_common_words.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/most_common_words.txt new file mode 100644 index 00000000..c72c72ce --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/most_common_words.txt @@ -0,0 +1,100 @@ +the +be +to +of +and +a +in +that +have +I +it +for +not +on +with +he +as +you +do +at +this +but +his +by +from +they +we +say +her +she +or +an +will +my +one +all +would +there +their +what +so +up +out +if +about +who +get +which +go +me +when +make +can +like +time +no +just +him +know +take +people +into +year +your +good +some +could +them +see +other +than +then +now +look +only +come +its +over +think +also +back +after +use +two +how +our +work +first +well +way +even +new +want +because +any +these +give +day +most +us \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 570c5ec8..76c80801 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -105,7 +104,7 @@ rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3"); dictionary.recommendEverywhere(entity); }); end @@ -118,7 +117,7 @@ rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(entity); }); end @@ -128,7 +127,7 @@ rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.3", "Author found", "personal_data_geolocation_article_39e3"); end rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" @@ -136,7 +135,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.4", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -154,7 +153,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then - $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.1.1", "Address found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -179,7 +178,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" @@ -192,7 +191,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" @@ -220,7 +219,7 @@ rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)" @@ -233,7 +232,7 @@ rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate st .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "CBI.10.2: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" @@ -380,7 +379,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -405,7 +404,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3")); end rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" @@ -414,7 +413,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -433,7 +432,7 @@ rule "PII.0.1: Redact all PII (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3"); end rule "PII.0.2: Redact all PII (vertebrate study)" @@ -441,7 +440,7 @@ rule "PII.0.2: Redact all PII (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "PII.0.3: Redact all PII" @@ -467,7 +466,7 @@ rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "personal_data_geolocation_article_39e3")); end rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" @@ -476,7 +475,7 @@ rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.1.5: Redact Emails by RegEx" @@ -528,7 +527,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "personal_data_geolocation_article_39e3")); end rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" @@ -545,7 +544,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -564,7 +563,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" @@ -573,7 +572,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" @@ -582,7 +581,7 @@ rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" @@ -591,7 +590,7 @@ rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -672,7 +671,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "personal_data_geolocation_article_39e3")); end rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)" @@ -685,7 +684,7 @@ rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -710,7 +709,7 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "personal_data_geolocation_article_39e3")); end rule "PII.6.2: Redact line between contact keywords (vertebrate study)" @@ -722,7 +721,7 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.6.3: Redact line between contact keywords (non vertebrate study)" @@ -766,7 +765,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "personal_data_geolocation_article_39e3")); end rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" @@ -784,7 +783,7 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -821,7 +820,7 @@ rule "PII.8.1: Redact contact information if producer is found (non vertebrate s entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "personal_data_geolocation_article_39e3")); end rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" @@ -839,7 +838,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -868,7 +867,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "personal_data_geolocation_article_39e3")); end @@ -878,7 +877,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "personal_data_geolocation_article_39e3")); end @@ -889,7 +888,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "personal_data_geolocation_article_39e3")); end rule "PII.12.1: Expand PII entities with salutation prefix" @@ -898,7 +897,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -927,7 +926,7 @@ rule "ETC.2.1: Redact signatures (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "personal_data_geolocation_article_39e3"); end rule "ETC.2.2: Redact signatures (vertebrate study)" @@ -935,7 +934,7 @@ rule "ETC.2.2: Redact signatures (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.2.3: Redact signatures" @@ -967,7 +966,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.3.3: Redact logos" @@ -985,7 +984,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -995,7 +993,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -1114,8 +1111,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1127,8 +1122,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -1139,10 +1132,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1152,9 +1143,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -1165,9 +1154,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -1178,8 +1165,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -1192,9 +1177,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1205,7 +1188,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1217,8 +1200,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -1239,7 +1220,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -1249,8 +1229,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -1261,71 +1240,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1333,68 +1355,100 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1420,25 +1474,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -1451,7 +1516,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 80a4e78f..625ca7d0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -118,7 +117,7 @@ rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3"); dictionary.recommendEverywhere(entity); }); end @@ -131,7 +130,7 @@ rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(entity); }); end @@ -141,7 +140,7 @@ rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.3", "Author found", "personal_data_geolocation_article_39e3"); end rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" @@ -149,7 +148,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.4", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -167,7 +166,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then - $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.1.1", "Address found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -232,7 +231,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" @@ -245,7 +244,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" @@ -286,7 +285,7 @@ rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)" @@ -299,7 +298,7 @@ rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate st .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "CBI.10.2: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" @@ -368,7 +367,7 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "personal_data_geolocation_article_39e3"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end @@ -386,7 +385,7 @@ rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); @@ -537,7 +536,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations" $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION) then $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); - retract($entity) end rule "CBI.13.1: Redacted because Section contains a vertebrate" @@ -765,7 +763,6 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials"); - retract($entityToExpand); }); end @@ -779,7 +776,6 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix"); - retract($entityToExpand); }); end @@ -818,7 +814,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -879,7 +875,7 @@ rule "CBI.21.2: Redact short Authors section (non vertebrate study)" then entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { - entity.redact("CBI.21.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.21.2", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3"); }); end @@ -891,7 +887,7 @@ rule "CBI.21.3: Redact short Authors section (vertebrate study)" then entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { - entity.redact("CBI.21.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.21.3", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2"); }); end @@ -915,7 +911,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3")); end rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" @@ -924,7 +920,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -943,7 +939,7 @@ rule "PII.0.1: Redact all PII (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3"); end rule "PII.0.2: Redact all PII (vertebrate study)" @@ -951,7 +947,7 @@ rule "PII.0.2: Redact all PII (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "PII.0.3: Redact all PII" @@ -977,7 +973,7 @@ rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "personal_data_geolocation_article_39e3")); end rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" @@ -986,7 +982,7 @@ rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.1.3: Redact typoed Emails with indicator" @@ -994,7 +990,7 @@ rule "PII.1.3: Redact typoed Emails with indicator" $section: Section(containsString("@") || containsStringIgnoreCase("mail")) then entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "personal_data_geolocation_article_39e3")); end rule "PII.1.4: Redact typoed Emails with indicator" @@ -1054,7 +1050,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "personal_data_geolocation_article_39e3")); end rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" @@ -1071,7 +1067,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.2.3: Redact phone numbers without indicators" @@ -1079,7 +1075,7 @@ rule "PII.2.3: Redact phone numbers without indicators" $section: Section(containsString("+")) then entityCreationService.byRegex("(\\+[\\dO]{1,2} )(\\([\\dO]{1,3}\\))?[\\d\\-O ]{8,15}", "PII", EntityType.ENTITY, $section) - .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "personal_data_geolocation_article_39e3")); end @@ -1098,7 +1094,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" @@ -1107,7 +1103,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.3.3: Redact telephone numbers by RegEx" @@ -1124,7 +1120,7 @@ rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" @@ -1133,7 +1129,7 @@ rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1214,7 +1210,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "personal_data_geolocation_article_39e3")); end rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)" @@ -1227,7 +1223,7 @@ rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1252,7 +1248,7 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "personal_data_geolocation_article_39e3")); end rule "PII.6.2: Redact line between contact keywords (vertebrate study)" @@ -1264,7 +1260,7 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.6.3: Redact line between contact keywords (non vertebrate study)" @@ -1308,7 +1304,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "personal_data_geolocation_article_39e3")); end rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" @@ -1326,7 +1322,7 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1363,7 +1359,7 @@ rule "PII.8.1: Redact contact information if producer is found (non vertebrate s entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "personal_data_geolocation_article_39e3")); end rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" @@ -1381,7 +1377,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1400,7 +1396,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3")); end rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" @@ -1409,7 +1405,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ver $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" @@ -1428,7 +1424,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "personal_data_geolocation_article_39e3")); end rule "PII.10.1: Redact study director abbreviation (vertebrate study)" @@ -1437,7 +1433,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1447,7 +1443,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "personal_data_geolocation_article_39e3")); end @@ -1458,7 +1454,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "personal_data_geolocation_article_39e3")); end rule "PII.12.1: Expand PII entities with salutation prefix" @@ -1467,7 +1463,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1525,7 +1521,7 @@ rule "ETC.2.1: Redact signatures (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "personal_data_geolocation_article_39e3"); end rule "ETC.2.2: Redact signatures (vertebrate study)" @@ -1533,7 +1529,7 @@ rule "ETC.2.2: Redact signatures (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.2.3: Redact signatures" @@ -1565,7 +1561,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.3.3: Redact logos" @@ -1588,21 +1584,21 @@ rule "ETC.4.0: Redact dossier dictionary entries" when $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "personal_data_geolocation_article_39e3"); end rule "ETC.4.1: Redact dossier dictionary entries" when $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.4.2: Redact dossier dictionary entries" when $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); + $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "links_producer_applicant"); end @@ -1613,7 +1609,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -1623,7 +1618,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -1658,7 +1652,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then - $logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.8.0", "Logo Found", "personal_data_geolocation_article_39e3"); end rule "ETC.8.1: Redact formulas (non vertebrate study)" @@ -1666,7 +1660,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then - $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.8.1", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -1705,7 +1699,7 @@ rule "ETC.11.0: Recommend first line in table cell with name and address of owne $tableCell: TableCell(col == $header.col, row == 2) from $table.streamTableCells().toList() then entityCreationService.bySemanticNode($tableCell, "PII", EntityType.RECOMMENDATION) - .ifPresent(redactionEntity -> redactionEntity.redact("ETC.11.0", "Trial Site owner and address found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .ifPresent(redactionEntity -> redactionEntity.redact("ETC.11.0", "Trial Site owner and address found", "personal_data_geolocation_article_39e3")); end @@ -1715,7 +1709,7 @@ rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.12.0", "Dossier dictionary entry found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.12.0", "Dossier dictionary entry found", "personal_data_geolocation_article_39e3"); end rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)" @@ -1723,7 +1717,7 @@ rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)" @@ -1881,8 +1875,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1894,8 +1886,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -1906,10 +1896,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1919,9 +1907,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -1932,9 +1918,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -1945,8 +1929,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -1959,9 +1941,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1972,7 +1952,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1984,8 +1964,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -2006,7 +1984,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -2016,8 +1993,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -2028,71 +2004,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -2100,68 +2119,100 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -2190,32 +2241,48 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -2228,7 +2295,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 8b3442c1..6f7b1efc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -1248,8 +1247,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1261,8 +1258,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -1273,10 +1268,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1286,9 +1279,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -1299,9 +1290,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -1312,8 +1301,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -1326,9 +1313,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1339,7 +1324,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1351,8 +1336,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -1373,7 +1356,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -1383,8 +1365,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -1395,71 +1376,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1467,22 +1491,33 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.7 @@ -1500,22 +1535,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1544,11 +1590,16 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end @@ -1561,7 +1612,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index 175b0355..d2bbd9db 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -105,7 +104,7 @@ rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3"); dictionary.recommendEverywhere(entity); }); end @@ -118,7 +117,7 @@ rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(entity); }); end @@ -128,7 +127,7 @@ rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.3", "Author found", "personal_data_geolocation_article_39e3"); end rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" @@ -136,7 +135,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.4", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -154,7 +153,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then - $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.1.1", "Address found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -179,7 +178,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" @@ -192,7 +191,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "personal_data_geolocation_article_39e3")); end @@ -207,7 +206,7 @@ rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)" @@ -220,7 +219,7 @@ rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate st .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -303,7 +302,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -316,7 +315,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3")); end rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" @@ -325,7 +324,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -337,7 +336,7 @@ rule "PII.0.1: Redact all PII (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3"); end rule "PII.0.2: Redact all PII (vertebrate study)" @@ -345,7 +344,7 @@ rule "PII.0.2: Redact all PII (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -356,7 +355,7 @@ rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "personal_data_geolocation_article_39e3")); end rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" @@ -365,7 +364,7 @@ rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.1.3: Redact typoed Emails with indicator" @@ -373,7 +372,7 @@ rule "PII.1.3: Redact typoed Emails with indicator" $section: Section(containsString("@") || containsStringIgnoreCase("mail")) then entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "personal_data_geolocation_article_39e3")); end @@ -392,7 +391,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "personal_data_geolocation_article_39e3")); end rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" @@ -409,7 +408,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.2.3: Redact phone numbers without indicators" @@ -417,7 +416,7 @@ rule "PII.2.3: Redact phone numbers without indicators" $section: Section(containsString("+")) then entityCreationService.byRegex("(\\+[\\dO]{1,2} )(\\([\\dO]{1,3}\\))?[\\d\\-O ]{8,15}", "PII", EntityType.ENTITY, $section) - .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "personal_data_geolocation_article_39e3")); end @@ -428,7 +427,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" @@ -437,7 +436,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" @@ -446,7 +445,7 @@ rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" @@ -455,7 +454,7 @@ rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -470,7 +469,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "personal_data_geolocation_article_39e3")); end rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)" @@ -483,7 +482,7 @@ rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -497,7 +496,7 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "personal_data_geolocation_article_39e3")); end rule "PII.6.2: Redact line between contact keywords (vertebrate study)" @@ -509,7 +508,7 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -529,7 +528,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "personal_data_geolocation_article_39e3")); end rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" @@ -547,7 +546,7 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -567,7 +566,7 @@ rule "PII.8.1: Redact contact information if producer is found (non vertebrate s entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "personal_data_geolocation_article_39e3")); end rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" @@ -585,7 +584,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -596,7 +595,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "personal_data_geolocation_article_39e3")); end rule "PII.10.1: Redact study director abbreviation (vertebrate study)" @@ -605,7 +604,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -615,7 +614,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "personal_data_geolocation_article_39e3")); end @@ -626,7 +625,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "personal_data_geolocation_article_39e3")); end rule "PII.12.1: Expand PII entities with salutation prefix" @@ -635,7 +634,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -657,7 +656,7 @@ rule "ETC.2.1: Redact signatures (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "personal_data_geolocation_article_39e3"); end rule "ETC.2.2: Redact signatures (vertebrate study)" @@ -665,7 +664,7 @@ rule "ETC.2.2: Redact signatures (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -683,7 +682,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -694,7 +693,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -704,7 +702,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -841,8 +838,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -854,8 +849,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -866,10 +859,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -879,9 +870,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -892,9 +881,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -905,8 +892,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -919,9 +904,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -932,7 +915,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -944,8 +927,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -966,7 +947,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -976,8 +956,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -988,71 +967,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1060,68 +1082,100 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1147,25 +1201,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -1178,7 +1243,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 47594a31..3988bd1a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -160,8 +159,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -173,8 +170,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -185,10 +180,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -198,9 +191,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -211,9 +202,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -224,8 +213,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -238,9 +225,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -251,7 +236,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -263,8 +248,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -285,7 +268,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -295,8 +277,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -307,71 +288,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -379,68 +403,100 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -466,25 +522,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -497,7 +564,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 991cbd8a..6580da5f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -18,8 +18,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -107,7 +109,7 @@ rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3"); dictionary.recommendEverywhere(entity); }); end @@ -120,7 +122,7 @@ rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(entity); }); end @@ -137,7 +139,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" @@ -150,7 +152,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" @@ -206,7 +208,7 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "personal_data_geolocation_article_39e3"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end @@ -224,7 +226,7 @@ rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); @@ -447,7 +449,6 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials"); - retract($entityToExpand); }); end @@ -461,7 +462,6 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix"); - retract($entityToExpand); }); end @@ -500,7 +500,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -525,7 +525,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3")); end rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" @@ -534,7 +534,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -553,7 +553,7 @@ rule "PII.0.1: Redact all PII (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3"); end rule "PII.0.2: Redact all PII (vertebrate study)" @@ -561,7 +561,7 @@ rule "PII.0.2: Redact all PII (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "PII.0.3: Redact all PII" @@ -587,7 +587,7 @@ rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "personal_data_geolocation_article_39e3")); end rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" @@ -596,7 +596,7 @@ rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.1.5: Redact Emails by RegEx" @@ -691,7 +691,7 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "personal_data_geolocation_article_39e3")); end rule "PII.6.2: Redact line between contact keywords (vertebrate study)" @@ -703,7 +703,7 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.6.3: Redact line between contact keywords (non vertebrate study)" @@ -747,7 +747,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "personal_data_geolocation_article_39e3")); end rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" @@ -765,7 +765,7 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -802,7 +802,7 @@ rule "PII.8.1: Redact contact information if producer is found (non vertebrate s entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "personal_data_geolocation_article_39e3")); end rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" @@ -820,7 +820,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -848,7 +848,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "personal_data_geolocation_article_39e3")); end @@ -859,7 +859,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "personal_data_geolocation_article_39e3")); end rule "PII.12.1: Expand PII entities with salutation prefix" @@ -868,7 +868,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -897,7 +897,7 @@ rule "ETC.2.1: Redact signatures (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "personal_data_geolocation_article_39e3"); end rule "ETC.2.2: Redact signatures (vertebrate study)" @@ -905,7 +905,7 @@ rule "ETC.2.2: Redact signatures (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.2.3: Redact signatures" @@ -937,7 +937,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.3.3: Redact logos" @@ -953,7 +953,7 @@ rule "ETC.4.0: Redact dossier dictionary entries" when $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "personal_data_geolocation_article_39e3"); end @@ -964,7 +964,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -974,7 +973,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -1009,7 +1007,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then - $logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.8.0", "Logo Found", "personal_data_geolocation_article_39e3"); end rule "ETC.8.1: Redact formulas (non vertebrate study)" @@ -1017,7 +1015,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then - $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.8.1", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -1136,8 +1134,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1149,8 +1145,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -1161,10 +1155,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1174,9 +1166,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -1187,9 +1177,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -1200,8 +1188,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -1214,9 +1200,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1227,7 +1211,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1239,8 +1223,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -1261,7 +1243,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -1271,8 +1252,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -1283,71 +1263,115 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end // Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1355,68 +1379,100 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1442,25 +1498,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -1473,7 +1540,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 17e1c255..d392e0bd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -79,7 +78,7 @@ rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3"); dictionary.recommendEverywhere(entity); }); end @@ -92,7 +91,7 @@ rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(entity); }); end @@ -102,7 +101,7 @@ rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.3", "Author found", "personal_data_geolocation_article_39e3"); end rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" @@ -110,7 +109,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.4", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -122,7 +121,7 @@ rule "PII.0.1: Redact all PII (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3"); end @@ -231,8 +230,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -244,8 +241,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -256,10 +251,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -269,9 +262,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -282,9 +273,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -295,8 +284,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -309,9 +296,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -322,7 +307,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -343,7 +328,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -353,8 +337,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -365,71 +348,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -437,68 +463,100 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -524,25 +582,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -555,7 +624,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 0166e897..2c3027ee 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -310,8 +309,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -323,8 +320,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -335,10 +330,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -348,9 +341,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -361,9 +352,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -374,8 +363,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -388,9 +375,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -401,7 +386,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -413,8 +398,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -435,7 +418,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -445,8 +427,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -457,71 +438,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -529,46 +553,67 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.7 @@ -586,22 +631,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -627,25 +683,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -658,7 +725,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 9c5c810d..7aa4c76d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -210,8 +209,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -223,8 +220,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -235,10 +230,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -248,9 +241,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -261,9 +252,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -274,8 +263,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -288,9 +275,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -301,7 +286,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -313,8 +298,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -335,7 +318,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -345,8 +327,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -357,92 +338,146 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -450,46 +485,67 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.7 @@ -507,22 +563,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -548,25 +615,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -579,7 +657,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/def8f960580f088b975ba806dfae1f87.REDACTION_LOG.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/def8f960580f088b975ba806dfae1f87.REDACTION_LOG.json index b8c2ff4e..c4f1bd45 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/def8f960580f088b975ba806dfae1f87.REDACTION_LOG.json +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/def8f960580f088b975ba806dfae1f87.REDACTION_LOG.json @@ -65,7 +65,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -121,7 +121,7 @@ "reason": "PII (Personal Identification Information) found", "matchedRule": 12, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -325,7 +325,7 @@ "reason": "PII (Personal Identification Information) found", "matchedRule": 12, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -381,7 +381,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -437,7 +437,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -549,7 +549,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -605,7 +605,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -661,7 +661,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -997,7 +997,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1053,7 +1053,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -1165,7 +1165,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1221,7 +1221,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1277,7 +1277,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1333,7 +1333,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1445,7 +1445,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1557,7 +1557,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1613,7 +1613,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1669,7 +1669,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -1893,7 +1893,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -2117,7 +2117,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -2285,7 +2285,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -2677,7 +2677,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -2789,7 +2789,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -2845,7 +2845,7 @@ "reason": "PII (Personal Identification Information) found", "matchedRule": 12, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -2957,7 +2957,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -3125,7 +3125,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -3237,7 +3237,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -3349,7 +3349,7 @@ "reason": "PII (Personal Identification Information) found", "matchedRule": 12, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -3405,7 +3405,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -3461,7 +3461,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -3629,7 +3629,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -3685,7 +3685,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -3797,7 +3797,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -3909,7 +3909,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -4021,7 +4021,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -4077,7 +4077,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -4189,7 +4189,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -4357,7 +4357,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -4413,7 +4413,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -4469,7 +4469,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -4525,7 +4525,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -4581,7 +4581,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -4637,7 +4637,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -4749,7 +4749,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -4917,7 +4917,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5141,7 +5141,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5253,7 +5253,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5309,7 +5309,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5365,7 +5365,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5421,7 +5421,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -5477,7 +5477,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5533,7 +5533,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -5589,7 +5589,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5645,7 +5645,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -5757,7 +5757,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -5813,7 +5813,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5878,7 +5878,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -5934,7 +5934,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -5990,7 +5990,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6046,7 +6046,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6102,7 +6102,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -6158,7 +6158,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -6214,7 +6214,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6279,7 +6279,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6335,7 +6335,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -6391,7 +6391,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6456,7 +6456,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6512,7 +6512,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6624,7 +6624,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6680,7 +6680,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -6801,7 +6801,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6857,7 +6857,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -6913,7 +6913,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -6969,7 +6969,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7025,7 +7025,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7081,7 +7081,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7137,7 +7137,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7193,7 +7193,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7249,7 +7249,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7305,7 +7305,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -7361,7 +7361,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7473,7 +7473,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -7529,7 +7529,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7585,7 +7585,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7641,7 +7641,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -7697,7 +7697,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -7753,7 +7753,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -7809,7 +7809,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7865,7 +7865,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -7977,7 +7977,7 @@ "reason": "Published Information found", "matchedRule": 11, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": false, "section": "", @@ -8145,7 +8145,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "Footer", @@ -8313,7 +8313,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -8481,7 +8481,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -8705,7 +8705,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -8761,7 +8761,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -8826,7 +8826,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -8882,7 +8882,7 @@ "reason": "Author found", "matchedRule": 1, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -8938,7 +8938,7 @@ "reason": "Address found", "matchedRule": 2, "rectangle": false, - "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002", + "legalBasis": "personal_data_geolocation_article_39e3", "imported": false, "redacted": true, "section": "", @@ -9046,49 +9046,58 @@ ], "legalBasis": [ { - "name": "1.1 personal data (incl. geolocation); Article 39(e)(1)", - "description": "any other personal data except for\n a. the name and address of the applicant;\n b. the names of authors of published or publicly available studies supporting such requests; and the names of all participants and observers in meetings of the Scientific Committee and the Scientific Panels, their working groups and any other ad hoc group meeting on the subject matter.", - "reason": "Article 39(e)(3) of Regulation (EC) No 178/2002" + "name": "1.1 personal data (incl. geolocation); Article 39(e)(3)", + "description": "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)", + "reason": "personal_data_geolocation_article_39e3", + "technicalName": "personal_data_geolocation_article_39e3" }, { "name": "1.2 vertebrate study related personal data (incl. geolocation); Article 39(e)(2)", "description": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information", - "reason": "Article 39(e)(2) of Regulation (EC) No 178/2002" + "reason": "Article 39(e)(2) of Regulation (EC) No 178/2002", + "technicalName": "vertebrate_study_personal_data_geolocation_article_39e2" }, { "name": "2. manufacturing or production process", "description": "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety", - "reason": "manufacturing_production_process" + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)", + "technicalName": "manufacturing_production_process" }, { "name": "3. links between a producer and applicant", - "description": "commercial links between a producer or importer and the applicant or the authorisation holder, where applicable;", - "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + "description": "commercial links between a producer or importer and the applicant or the authorisation holder, where applicable", + "reason": "links_producer_applicant", + "technicalName": "links_producer_applicant" }, { "name": "4. commercial information", "description": "commercial information revealing sourcing, market shares or business strategy of the applicant", - "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)", + "technicalName": "commercial_information" }, { "name": "5. quantitative composition", "description": "quantitative composition of the subject matter of the request, except for information which is relevant to the assessment of safety", - "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)" + "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)", + "technicalName": "quantitative_composition" }, { "name": "6. specification of impurity", "description": "the specification of impurity of the active substance and the related methods of analysis for impurities in the active substance as manufactured, except for the impurities that are considered to be toxicologically, ecotoxicologically or environmentally relevant and the related methods of analysis for such impurities", - "reason": "specification_impurity" + "reason": "Article 63(2)(b) of Regulation (EC) No 1107/2009", + "technicalName": "specification_impurity" }, { "name": "7. results of production batches", "description": "results of production batches of the active substance including impurities", - "reason": "results_production_batches" + "reason": "Article 63(2)(c) of Regulation (EC) No 1107/2009", + "technicalName": "results_production_batches" }, { "name": "8. composition of a plant protection product", "description": "information on the complete composition of a plant protection product", - "reason": "composition_plant_protection_product" + "reason": "Article 63(2)(d) of Regulation (EC) No 1107/2009", + "technicalName": "composition_plant_protection_product" } ], "dictionaryVersion": 492, diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml index 8e3e4cf2..03547171 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml @@ -16,5 +16,7 @@ + + \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index 96eaf8e6..fcb55325 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -94,7 +93,7 @@ rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3"); dictionary.recommendEverywhere(entity); }); end @@ -107,7 +106,7 @@ rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(entity); }); end @@ -117,7 +116,7 @@ rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.apply("CBI.0.0", "Author found", "personal_data_geolocation_article_39e3"); end rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" @@ -125,7 +124,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.4", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -143,7 +142,7 @@ rule "CBI.1.1: Redact CBI Address (Vertebrate Study)" FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then - $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.1.1", "Address found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -168,7 +167,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" @@ -181,7 +180,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "personal_data_geolocation_article_39e3")); end @@ -196,7 +195,7 @@ rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)" @@ -209,7 +208,7 @@ rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate st .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -292,7 +291,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -305,7 +304,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3")); end rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" @@ -314,7 +313,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -326,7 +325,7 @@ rule "PII.0.1: Redact all PII (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3"); end rule "PII.0.2: Redact all PII (vertebrate study)" @@ -334,7 +333,7 @@ rule "PII.0.2: Redact all PII (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -345,7 +344,7 @@ rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "personal_data_geolocation_article_39e3")); end rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" @@ -354,7 +353,7 @@ rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.1.3: Redact typoed Emails with indicator" @@ -362,7 +361,7 @@ rule "PII.1.3: Redact typoed Emails with indicator" $section: Section(containsString("@") || containsStringIgnoreCase("mail")) then entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "personal_data_geolocation_article_39e3")); end @@ -381,7 +380,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "personal_data_geolocation_article_39e3")); end rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" @@ -398,7 +397,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -409,7 +408,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" @@ -418,7 +417,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" @@ -427,7 +426,7 @@ rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" @@ -436,7 +435,7 @@ rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -451,7 +450,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "personal_data_geolocation_article_39e3")); end rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)" @@ -464,7 +463,7 @@ rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -478,7 +477,7 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "personal_data_geolocation_article_39e3")); end rule "PII.6.2: Redact line between contact keywords (vertebrate study)" @@ -490,7 +489,7 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -510,7 +509,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "personal_data_geolocation_article_39e3")); end rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" @@ -528,7 +527,7 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -548,7 +547,7 @@ rule "PII.8.1: Redact contact information if producer is found (non vertebrate s entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "personal_data_geolocation_article_39e3")); end rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" @@ -566,7 +565,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -577,7 +576,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "personal_data_geolocation_article_39e3")); end rule "PII.10.1: Redact study director abbreviation (vertebrate study)" @@ -586,7 +585,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -596,7 +595,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "personal_data_geolocation_article_39e3")); end @@ -607,7 +606,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "personal_data_geolocation_article_39e3")); end rule "PII.12.1: Expand PII entities with salutation prefix" @@ -616,7 +615,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -638,7 +637,7 @@ rule "ETC.2.1: Redact signatures (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "personal_data_geolocation_article_39e3"); end rule "ETC.2.2: Redact signatures (vertebrate study)" @@ -646,7 +645,7 @@ rule "ETC.2.2: Redact signatures (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -664,7 +663,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -723,8 +722,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -736,8 +733,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -748,10 +743,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -761,9 +754,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -774,9 +765,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -787,8 +776,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -801,9 +788,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -814,7 +799,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -826,8 +811,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -848,7 +831,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -858,8 +840,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -870,71 +851,114 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -942,68 +966,100 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1029,25 +1085,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -1060,7 +1127,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 648fdf20..9a49cd37 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection; @@ -118,7 +117,7 @@ rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3"); dictionary.recommendEverywhere(entity); }); end @@ -131,7 +130,7 @@ rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { - entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(entity); }); end @@ -141,7 +140,7 @@ rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.3", "Author found", "personal_data_geolocation_article_39e3"); end rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" @@ -149,7 +148,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_author", dictionaryEntry) then - $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.0.4", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -167,7 +166,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type() == "CBI_address", dictionaryEntry) then - $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.redact("CBI.1.1", "Address found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -232,7 +231,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" @@ -245,7 +244,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "personal_data_geolocation_article_39e3")); end rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" @@ -285,7 +284,7 @@ rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)" @@ -298,7 +297,7 @@ rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate st .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "CBI.10.2: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" @@ -367,7 +366,7 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s then entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "personal_data_geolocation_article_39e3"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); end @@ -385,7 +384,7 @@ rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) .ifPresent(authorEntity -> { - authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.addMultipleAuthorsAsRecommendation(authorEntity); }); @@ -541,7 +540,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations" $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION) then $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); - retract($entity) end // from CBI.3.0 @@ -775,7 +773,6 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials"); - retract($entityToExpand); }); end @@ -789,7 +786,6 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix"); - retract($entityToExpand); }); end @@ -828,7 +824,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { - laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "vertebrate_study_personal_data_geolocation_article_39e2"); dictionary.recommendEverywhere(laboratoryEntity); }); end @@ -891,7 +887,7 @@ rule "CBI.21.2: Redact short Authors section (non vertebrate study)" then entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { - entity.redact("CBI.21.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.redact("CBI.21.2", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3"); }); end @@ -903,7 +899,7 @@ rule "CBI.21.3: Redact short Authors section (vertebrate study)" then entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section) .forEach(entity -> { - entity.redact("CBI.21.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.redact("CBI.21.3", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2"); }); end @@ -927,7 +923,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3")); end rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" @@ -936,7 +932,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200) - .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -955,7 +951,7 @@ rule "PII.0.1: Redact all PII (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3"); end rule "PII.0.2: Redact all PII (vertebrate study)" @@ -963,7 +959,7 @@ rule "PII.0.2: Redact all PII (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type() == "PII", dictionaryEntry) then - $pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $pii.redact("PII.0.2", "Personal Information found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "PII.0.3: Redact all PII" @@ -989,7 +985,7 @@ rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "personal_data_geolocation_article_39e3")); end rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" @@ -998,7 +994,7 @@ rule "PII.1.2: Redact Emails by RegEx (vertebrate study)" $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.1.3: Redact typoed Emails with indicator" @@ -1006,7 +1002,7 @@ rule "PII.1.3: Redact typoed Emails with indicator" $section: Section(containsString("@") || containsStringIgnoreCase("mail")) then entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section) - .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "personal_data_geolocation_article_39e3")); end rule "PII.1.4: Redact typoed Emails with indicator" @@ -1066,7 +1062,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "personal_data_geolocation_article_39e3")); end rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" @@ -1083,7 +1079,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)" containsString("Fer")) then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) - .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.2.3: Redact phone numbers without indicators" @@ -1091,7 +1087,7 @@ rule "PII.2.3: Redact phone numbers without indicators" $section: Section(containsString("+")) then entityCreationService.byRegex("(\\+[\\dO]{1,2} )(\\([\\dO]{1,3}\\))?[\\d\\-O ]{8,15}", "PII", EntityType.ENTITY, $section) - .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "personal_data_geolocation_article_39e3")); end @@ -1110,7 +1106,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" @@ -1119,7 +1115,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)" $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.3.3: Redact telephone numbers by RegEx" @@ -1136,7 +1132,7 @@ rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "personal_data_geolocation_article_39e3")); end rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" @@ -1145,7 +1141,7 @@ rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)" $rowCell: TableCell(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell) - .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1226,7 +1222,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (non verte $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "personal_data_geolocation_article_39e3")); end rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)" @@ -1239,7 +1235,7 @@ rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrat $section: Section(containsString($contactKeyword)) then entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) - .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1264,7 +1260,7 @@ rule "PII.6.1: Redact line between contact keywords (non vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "personal_data_geolocation_article_39e3")); end rule "PII.6.2: Redact line between contact keywords (vertebrate study)" @@ -1276,7 +1272,7 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)" entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) ) - .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "vertebrate_study_personal_data_geolocation_article_39e2")); end rule "PII.6.3: Redact line between contact keywords (non vertebrate study)" @@ -1320,7 +1316,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "personal_data_geolocation_article_39e3")); end rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)" @@ -1338,7 +1334,7 @@ rule "PII.7.2: Redact contact information if applicant is found (vertebrate stud entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1375,7 +1371,7 @@ rule "PII.8.1: Redact contact information if producer is found (non vertebrate s entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "personal_data_geolocation_article_39e3")); end rule "PII.8.2: Redact contact information if producer is found (vertebrate study)" @@ -1393,7 +1389,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section), entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section) )) - .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1413,7 +1409,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3")); end rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" @@ -1422,7 +1418,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ver $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1442,7 +1438,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "personal_data_geolocation_article_39e3")); end @@ -1452,7 +1448,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)" $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML")) then entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section) - .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1462,7 +1458,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:" $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) then entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "personal_data_geolocation_article_39e3")); end @@ -1473,7 +1469,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "personal_data_geolocation_article_39e3")); end @@ -1484,7 +1480,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix" $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") - .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "vertebrate_study_personal_data_geolocation_article_39e2")); end @@ -1542,7 +1538,7 @@ rule "ETC.2.1: Redact signatures (non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.1", "Signature Found", "personal_data_geolocation_article_39e3"); end rule "ETC.2.2: Redact signatures (vertebrate study)" @@ -1550,7 +1546,7 @@ rule "ETC.2.2: Redact signatures (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then - $signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $signature.redact("ETC.2.2", "Signature Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.2.3: Redact signatures" @@ -1582,7 +1578,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then - $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.3.2", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.3.3: Redact logos" @@ -1606,21 +1602,21 @@ rule "ETC.4.0: Redact dossier dictionary entries" when $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "personal_data_geolocation_article_39e3"); end rule "ETC.4.1: Redact dossier dictionary entries" when $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.4.2: Redact dossier dictionary entries" when $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"); + $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "links_producer_applicant"); end @@ -1631,7 +1627,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -1641,7 +1636,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -1676,7 +1670,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then - $logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.8.0", "Logo Found", "personal_data_geolocation_article_39e3"); end rule "ETC.8.1: Redact formulas (non vertebrate study)" @@ -1684,7 +1678,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then - $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $logo.redact("ETC.8.1", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2"); end @@ -1723,7 +1717,7 @@ rule "ETC.11.0: Recommend first line in table cell with name and address of owne $tableCell: TableCell(col == $header.col, row == 2) from $table.streamTableCells().toList() then entityCreationService.bySemanticNode($tableCell, "PII", EntityType.RECOMMENDATION) - .ifPresent(redactionEntity -> redactionEntity.redact("ETC.11.0", "Trial Site owner and address found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + .ifPresent(redactionEntity -> redactionEntity.redact("ETC.11.0", "Trial Site owner and address found", "personal_data_geolocation_article_39e3")); end @@ -1733,7 +1727,7 @@ rule "ETC.12.0: Redact dossier_redaction (Non vertebrate study)" not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.12.0", "Dossier dictionary entry found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.12.0", "Dossier dictionary entry found", "personal_data_geolocation_article_39e3"); end rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)" @@ -1741,7 +1735,7 @@ rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)" FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $dossierRedaction: TextEntity(type() == "dossier_redaction") then - $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "vertebrate_study_personal_data_geolocation_article_39e2"); end rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)" @@ -1899,8 +1893,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1912,8 +1904,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -1924,10 +1914,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1937,9 +1925,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -1950,9 +1936,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -1963,8 +1947,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -1977,9 +1959,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1990,7 +1970,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -2002,8 +1982,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -2025,7 +2003,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -2035,8 +2012,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -2047,81 +2023,134 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end + rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end + rule "X.0.4: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then $contained.remove("X.0.4", "remove Entity contained by Entity of same type"); - update($contained); end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -2129,68 +2158,100 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())), + !$contained.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT), + $container.active(), + $contained.type() != $container.type(), + eval($container.getTextRange().length() > $contained.getTextRange().length()), + !$contained.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); - end + $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); +end // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -2219,34 +2280,50 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -2259,7 +2336,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index 3eac9dd4..1a2d6cb5 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -18,8 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; -import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity -import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; @@ -1315,8 +1314,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1328,8 +1325,6 @@ rule "MAN.0.1: Apply manual resize redaction" then manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); retract($resizeRedaction); - update($imageToBeResized); - update($imageToBeResized.getParent()); end @@ -1340,10 +1335,8 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers) $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then - $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); + $entityToBeRemoved.addManualChange($idRemoval); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1353,9 +1346,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($imageEntityToBeRemoved); retract($idRemoval); - update($imageEntityToBeRemoved.getParent()); end @@ -1366,9 +1357,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId) $entityToForce: TextEntity(matchesAnnotationId($id)) then - $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + $entityToForce.addManualChange($force); retract($force); end @@ -1379,8 +1368,6 @@ rule "MAN.2.1: Apply force redaction to images" $imageToForce: Image(id == $id) then $imageToForce.getManualOverwrite().addChange($force); - update($imageToForce); - update($imageToForce.getParent()); retract($force); end @@ -1393,9 +1380,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1406,7 +1391,7 @@ rule "MAN.3.1: Apply entity recategorization of same type" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type) then - $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + $entityToBeRecategorized.addManualChange($recategorization); retract($recategorization); end @@ -1418,8 +1403,6 @@ rule "MAN.3.2: Apply image recategorization" $imageToBeRecategorized: Image($id == id) then manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); - update($imageToBeRecategorized); - update($imageToBeRecategorized.getParent()); retract($recategorization); end @@ -1439,7 +1422,6 @@ rule "MAN.4.0: Apply legal basis change" $imageToBeRecategorized: Image($id == id) then $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange); - update($imageToBeRecategorized) retract($legalBasisChange) end @@ -1449,8 +1431,7 @@ rule "MAN.4.1: Apply legal basis change" $legalBasisChange: ManualLegalBasisChange($id: annotationId) $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then - $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) + $entityToBeChanged.addManualChange($legalBasisChange); retract($legalBasisChange) end @@ -1461,72 +1442,115 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + !$container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) + not TextEntity( + getTextRange().equals($container.getTextRange()), + type() == $container.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); - end +end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.type() == $contained.type(), + $container.entityType == $contained.entityType, + $container != $contained, + !$container.removed(), + $container.hasManualChanges(), + !$contained.hasManualChanges(), + !$contained.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); - end +end // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.ENTITY, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); +end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_POSITIVE, + $container.active(), + $contained.entityType == EntityType.HINT, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) - end + $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); +end -// Rule unit: X.3 -rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" +rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.FALSE_RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $contained.type() == $container.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); +end // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1534,11 +1558,16 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end @@ -1546,12 +1575,18 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $container: container, + $contained: contained, + $container.entityType == EntityType.RECOMMENDATION, + $container.active(), + $contained.entityType == EntityType.RECOMMENDATION, + $container.type() != $contained.type(), + !$contained.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); - end + $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); +end // Rule unit: X.7 @@ -1569,22 +1604,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end // Rule unit: X.9 @@ -1611,11 +1657,16 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end @@ -1628,7 +1679,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv b/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv index 3b704b80..e69de29b 100644 --- a/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv +++ b/redaction-service-v1/rules-management/src/main/resources/old_rules_with_translations.csv @@ -1,240 +0,0 @@ -id,old rule names,old rule code,translates to -0,"""0: Add CBI_author from ai 5""","""\n when\n Section(aiMatchesType(\""CARDINAL\""))\n then\n section.addAiEntities(\""CARDINAL\"", \""cardinal\"");\n end""",[AI.2.0] -1,"""0: Add CBI_author from ai 3""","""\n when\n Section(aiMatchesType(\""POSTAL\""))\n then\n section.addAiEntities(\""POSTAL\"", \""postal\"");\n end""",[AI.2.0] -2,"""0: Add CBI_author from ai 6""","""\n when\n Section(aiMatchesType(\""CITY\""))\n then\n section.addAiEntities(\""CITY\"", \""city\"");\n end""",[AI.2.0] -3,"""0: Add CBI_author from ai 7""","""\n when\n Section(aiMatchesType(\""STATE\""))\n then\n section.addAiEntities(\""STATE\"", \""state\"");\n end""",[AI.2.0] -4,"""0: Add CBI_author from ai 2""","""\n when\n Section(aiMatchesType(\""STREET\""))\n then\n section.addAiEntities(\""STREET\"", \""street\"");\n end""",[AI.2.0] -5,"""0: Recommend CTL/BL laboratory that start with BL or CTL""","""\n when\n Section(searchText.contains(\""CT\"") || searchText.contains(\""BL\""))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n section.addRecommendationByRegEx(\""((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\"", true, 0, \""CBI_address\"");\n end""",[SYN.1.0] -6,"""0: Add CBI_author from ai 4""","""\n when\n Section(aiMatchesType(\""COUNTRY\""))\n then\n section.addAiEntities(\""COUNTRY\"", \""country\"");\n end""",[AI.2.0] -7,"""0: Combine address parts from ai to CBI_address (org is mandatory)""","""\n when\n Section(aiMatchesType(\""ORG\""))\n then\n section.combineAiTypes(\""ORG\"", \""STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\"", 20, \""CBI_address\"", 3, false);\n end""",[AI.1.0] -8,"""0: Combine address parts from ai to CBI_address (city is mandatory)""","""\n when\n Section(aiMatchesType(\""CITY\""))\n then\n section.combineAiTypes(\""CITY\"", \""ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\"", 20, \""CBI_address\"", 3, false);\n end""",[AI.1.0] -9,"""0: Combine address parts from ai to CBI_address (street is mandatory)""","""\n when\n Section(aiMatchesType(\""STREET\""))\n then\n section.combineAiTypes(\""STREET\"", \""ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\"", 20, \""CBI_address\"", 3, false);\n end""",[AI.1.0] -10,"""0: Add CBI_author from ai, 1: Add CBI_author from ai""","""\n when\n Section(aiMatchesType(\""CBI_author\""))\n then\n section.addAiEntities(\""CBI_author\"", \""CBI_author\"");\n end""",[AI.0.0] -11,"""0: Add CBI_author from ai 8""","""\n when\n Section(aiMatchesType(\""ORG\""))\n then\n section.addAiEntities(\""ORG\"", \""org\"");\n end""",[AI.2.0] -12,"""1: Redacted because Section contains Vertebrate""","""\n when\n Section(matchesType(\""vertebrate\""))\n then\n section.redact(\""CBI_author\"", 1, \""Vertebrate found\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 1, \""Vertebrate found\"", \""names_addresses_persons\"");\n end""",[CBI.3.0] -13,"""5: Do not redact genitive CBI_author, 7: Do not redact genitive CBI_author, 1: Do not redact genitive CBI_author""","""\n when\n Section(matchesType(\""CBI_author\""))\n then\n section.expandToFalsePositiveByRegEx(\""CBI_author\"", \""['’’'ʼˈ´`‘′ʻ’']s\"", false, 0);\n end""",[CBI.2.0] -14,"""1: Redact CBI Authors (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_author\""))\n then\n section.redact(\""CBI_author\"", 1, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[CBI.0.0] -15,"""2: Combine ai types CBI_author from ai""","""\n when\n Section(aiMatchesType(\""ORG\""))\n then\n section.combineAiTypes(\""ORG\"", \""STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\"", 20, \""CBI_address\"", 2, false);\n end""",[AI.1.0] -16,"""2: Not Redacted because Section contains no Vertebrate""","""\n when\n Section(!matchesType(\""vertebrate\""))\n then\n section.redactNot(\""CBI_author\"", 2, \""No Vertebrate found\"");\n section.redactNot(\""CBI_address\"", 2, \""No Vertebrate found\"");\n end""",[CBI.3.0] -17,"""2: Redact CBI Authors (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_author\""))\n then\n section.redact(\""CBI_author\"", 2, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.0.0] -18,"""2: Redacted because Section contains Vertebrate""","""\n when\n Section(matchesType(\""vertebrate\""))\n then\n section.redact(\""CBI_author\"", 2, \""Vertebrate found\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 2, \""Vertebrate found\"", \""names_addresses_persons\"");\n end""",[CBI.3.0] -19,"""3: Redact CBI Authors (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_author\""))\n then\n section.redact(\""CBI_author\"", 3, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[CBI.0.0] -20,"""3: Not Redacted because Section contains no Vertebrate""","""\n when\n Section(!matchesType(\""vertebrate\""))\n then\n section.redactNot(\""CBI_author\"", 3, \""No Vertebrate found\"");\n section.redactNot(\""CBI_address\"", 3, \""No Vertebrate found\"");\n end""",[CBI.3.2] -21,"""3: Do not redact Names and Addresses if no redaction Indicator is contained""","""\n when\n Section(matchesType(\""vertebrate\""), matchesType(\""no_redaction_indicator\""))\n then\n section.redactNot(\""CBI_author\"", 3, \""Vertebrate and No Redaction Indicator found\"");\n section.redactNot(\""CBI_address\"", 3, \""Vertebrate and No Redaction Indicator found\"");\n end""",[CBI.4.0] -22,"""3: Redact not CBI Address (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_address\""))\n then\n section.redactNot(\""CBI_address\"", 3, \""Address found for non vertebrate study\"");\n end""",[CBI.1.0] -23,"""3: Redact not CBI Address (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_address\""))\n then\n section.redactNot(\""CBI_address\"", 3, \""Address found for non vertebrate study\"");\n section.ignoreRecommendations(\""CBI_address\"");\n end""",[CBI.1.0] -24,"""4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained""","""\n when\n Section(matchesType(\""vertebrate\""), matchesType(\""no_redaction_indicator\""), matchesType(\""redaction_indicator\""))\n then\n section.redact(\""CBI_author\"", 4, \""Vertebrate and Redaction Indicator found\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 4, \""Vertebrate and Redaction Indicator found\"", \""names_addresses_persons\"");\n end""",[CBI.5.0] -25,"""4: Redact CBI Address (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_address\""))\n then\n section.redact(\""CBI_address\"", 4, \""Address found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.1.1] -26,"""4: Redact CBI Authors (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_author\""))\n then\n section.redact(\""CBI_author\"", 4, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.0.0] -27,"""4: Do not redact Names and Addresses if no redaction Indicator is contained""","""\n when\n Section(matchesType(\""vertebrate\""), matchesType(\""no_redaction_indicator\""))\n then\n section.redactNot(\""CBI_author\"", 4, \""Vertebrate and No Redaction Indicator found\"");\n section.redactNot(\""CBI_address\"", 4, \""Vertebrate and No Redaction Indicator found\"");\n end""",[CBI.4.0] -28,"""5: Do not redact Names and Addresses if no redaction Indicator is contained""","""\n when\n Section(matchesType(\""vertebrate\""), matchesType(\""published_information\""))\n then\n section.redactNotAndReference(\""CBI_author\"",\""published_information\"", 5, \""Published Information found\"");\n section.redactNotAndReference(\""CBI_address\"",\""published_information\"", 5, \""Published Information found\"");\n end""",[CBI.4.0] -29,"""5: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained""","""\n when\n Section(matchesType(\""vertebrate\""), matchesType(\""no_redaction_indicator\""), matchesType(\""redaction_indicator\""))\n then\n section.redact(\""CBI_author\"", 5, \""Vertebrate and Redaction Indicator found\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 5, \""Vertebrate and Redaction Indicator found\"", \""names_addresses_persons\"");\n end""",[CBI.5.0] -30,"""5: Redact not CBI Address (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_address\""))\n then\n section.redactNot(\""CBI_address\"", 5, \""Address found for non vertebrate study\"");\n section.ignoreRecommendations(\""CBI_address\"");\n end""",[CBI.1.0] -31,"""6: Do not redact Names and Addresses if no redaction Indicator is contained""","""\n when\n Section(matchesType(\""vertebrate\""), matchesType(\""published_information\""))\n then\n section.redactNotAndReference(\""CBI_author\"",\""published_information\"", 6, \""Published Information found\"");\n section.redactNotAndReference(\""CBI_address\"",\""published_information\"", 6, \""Published Information found\"");\n end""",[CBI.4.0] -32,"""6: Redact CBI Address (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""CBI_address\""))\n then\n section.redact(\""CBI_address\"", 6, \""Address found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.1.1] -33,"""6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && hasTableHeader(\""Author(s)\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author(s)\"", 6, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.11.0]" -34,"""6: Not redacted because Vertebrate Study = N""","""\n when\n Section(rowEquals(\""Vertebrate study Y/N\"", \""N\"") || rowEquals(\""Vertebrate study Y/N\"", \""No\""))\n then\n section.redactNotCell(\""Author(s)\"", 6, \""CBI_author\"", true, \""Not redacted because row is not a vertebrate study\"");\n section.redactNot(\""CBI_author\"", 6, \""Not redacted because row is not a vertebrate study\"");\n section.redactNot(\""CBI_address\"", 6, \""Not redacted because row is not a vertebrate study\"");\n section.highlightCell(\""Vertebrate study Y/N\"", 6, \""hint_only\"");\n end""",[CBI.12.0] -35,"""7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && hasTableHeader(\""Author(s)\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author(s)\"", 7, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.10.0, CBI.11.0]" -36,"""7: Not redacted because Vertebrate Study = N""","""\n when\n Section(rowEquals(\""Vertebrate study Y/N\"", \""N\"") || rowEquals(\""Vertebrate study Y/N\"", \""No\""))\n then\n section.redactNotCell(\""Author(s)\"", 7, \""CBI_author\"", true, \""Not redacted because row is not a vertebrate study\"");\n section.redactNot(\""CBI_author\"", 7, \""Not redacted because row is not a vertebrate study\"");\n section.redactNot(\""CBI_address\"", 7, \""Not redacted because row is not a vertebrate study\"");\n section.highlightCell(\""Vertebrate study Y/N\"", 7, \""hint_only\"");\n end""",[CBI.12.0] -37,"""7: Redact if must redact entry is found""","""\n when\n Section(matchesType(\""must_redact\""))\n then\n section.redact(\""CBI_author\"", 7, \""must_redact entry was found.\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 7, \""must_redact entry was found.\"", \""names_addresses_persons\"");\n end""",[CBI.8.0] -38,"""8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study""","""\n when\n Section(rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\""))\n then\n section.redactCell(\""Author(s)\"", 8, \""CBI_author\"", true, \""Redacted because row is a vertebrate study\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 8, \""Redacted because row is a vertebrate study\"", \""names_addresses_persons\"");\n section.highlightCell(\""Vertebrate study Y/N\"", 8, \""must_redact\"");\n end""",[CBI.12.0] -39,"""8: Redact if must redact entry is found""","""\n when\n Section(matchesType(\""must_redact\""))\n then\n section.redact(\""CBI_author\"", 8, \""Specification of impurity of the active substance was found.\"", \""specification_impurity_active_substance\"");\n section.redact(\""CBI_address\"", 8, \""Specification of impurity of the active substance was found.\"", \""specification_impurity_active_substance\"");\n end""",[CBI.8.0] -40,"""8: Redact Author cells in Tables with Author header (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && hasTableHeader(\""Author\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author\"", 8, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.11.0]" -41,"""8: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && hasTableHeader(\""Author(s)\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author(s)\"", 8, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.11.0]" -42,"""9: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && hasTableHeader(\""Author(s)\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author(s)\"", 9, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.10.0, CBI.11.0]" -43,"""9: Redact Author cells in Tables with Author header (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && hasTableHeader(\""Author\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author\"", 9, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.10.0, CBI.11.0]" -44,"""9: Redact sponsor company""","""\n when\n Section(searchText.toLowerCase().contains(\""batches produced at\""))\n then\n section.redactIfPrecededBy(\""batches produced at\"", \""CBI_sponsor\"", 9, \""Redacted because it represents a sponsor company\"", \""names_addresses_persons\"");\n section.addHintAnnotation(\""batches produced at\"", \""must_redact\"");\n end""",[CBI.14.0] -45,"""9: Redact Authors and Addresses in Reference Table if it is a Vertebrate study""","""\n when\n Section(rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\""))\n then\n section.redactCell(\""Author(s)\"", 9, \""CBI_author\"", true, \""Redacted because row is a vertebrate study\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 9, \""Redacted because row is a vertebrate study\"", \""names_addresses_persons\"");\n section.highlightCell(\""Vertebrate study Y/N\"", 9, \""must_redact\"");\n end""",[CBI.12.0] -46,"""10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\"") || rowEquals(\""Vertebrate study Y/N\"", \""N\"") || rowEquals(\""Vertebrate study Y/N\"", \""No\"")))\n then\n section.redactCell(\""Author(s)\"", 10, \""CBI_author\"", true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.11.0]" -47,"""10: Redact determination of residues""","""\n when\n Section((\n searchText.toLowerCase.contains(\""determination of residues\"") ||\n searchText.toLowerCase.contains(\""determination of total residues\"")\n ) && (\n searchText.toLowerCase.contains(\""livestock\"") ||\n searchText.toLowerCase.contains(\""live stock\"") ||\n searchText.toLowerCase.contains(\""tissue\"") ||\n searchText.toLowerCase.contains(\""tissues\"") ||\n searchText.toLowerCase.contains(\""liver\"") ||\n searchText.toLowerCase.contains(\""muscle\"") ||\n searchText.toLowerCase.contains(\""bovine\"") ||\n searchText.toLowerCase.contains(\""ruminant\"") ||\n searchText.toLowerCase.contains(\""ruminants\"")\n ))\n then\n section.redact(\""CBI_author\"", 10, \""Determination of residues was found.\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 10, \""Determination of residues was found.\"", \""names_addresses_persons\"");\n section.addHintAnnotation(\""determination of residues\"", \""must_redact\"");\n section.addHintAnnotation(\""determination of total residues\"", \""must_redact\"");\n section.addHintAnnotation(\""livestock\"", \""must_redact\"");\n section.addHintAnnotation(\""live stock\"", \""must_redact\"");\n section.addHintAnnotation(\""tissue\"", \""must_redact\"");\n section.addHintAnnotation(\""tissues\"", \""must_redact\"");\n section.addHintAnnotation(\""liver\"", \""must_redact\"");\n section.addHintAnnotation(\""muscle\"", \""must_redact\"");\n section.addHintAnnotation(\""bovine\"", \""must_redact\"");\n section.addHintAnnotation(\""ruminant\"", \""must_redact\"");\n section.addHintAnnotation(\""ruminants\"", \""must_redact\"");\n end""",[CBI.15.0] -48,"""10: Redact Author cells in Tables with Author header (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && hasTableHeader(\""Author\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author\"", 10, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.11.0]" -49,"""10: Redact sponsor company""","""\n when\n Section(searchText.toLowerCase().contains(\""batches produced at\""))\n then\n section.redactIfPrecededBy(\""batches produced at\"", \""CBI_sponsor\"", 10, \""Redacted because it represents a sponsor company\"", \""names_addresses_persons\"");\n section.addHintAnnotation(\""batches produced at\"", \""must_redact\"");\n end""",[CBI.14.0] -50,"""11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\"") || rowEquals(\""Vertebrate study Y/N\"", \""N\"") || rowEquals(\""Vertebrate study Y/N\"", \""No\"")))\n then\n section.redactCell(\""Author(s)\"", 11, \""CBI_author\"", true, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.10.0, CBI.11.0]" -51,"""11: Redact if CTL/* or BL/* was found""","""\n when\n Section(searchText.contains(\""CTL/\"") || searchText.contains(\""BL/\""))\n then\n section.redact(\""CBI_author\"", 11, \""Laboraty for vertebrate studies found\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 11, \""Laboraty for vertebrate studies found\"", \""names_addresses_persons\"");\n section.addHintAnnotation(\""CTL\"", \""must_redact\"");\n section.addHintAnnotation(\""BL\"", \""must_redact\"");\n end""",[SYN.0.0] -52,"""11: Redact determination of residues""","""\n when\n Section((\n searchText.toLowerCase.contains(\""determination of residues\"") ||\n searchText.toLowerCase.contains(\""determination of total residues\"")\n ) && (\n searchText.toLowerCase.contains(\""livestock\"") ||\n searchText.toLowerCase.contains(\""live stock\"") ||\n searchText.toLowerCase.contains(\""tissue\"") ||\n searchText.toLowerCase.contains(\""tissues\"") ||\n searchText.toLowerCase.contains(\""liver\"") ||\n searchText.toLowerCase.contains(\""muscle\"") ||\n searchText.toLowerCase.contains(\""bovine\"") ||\n searchText.toLowerCase.contains(\""ruminant\"") ||\n searchText.toLowerCase.contains(\""ruminants\"")\n ))\n then\n section.redact(\""CBI_author\"", 11, \""Determination of residues was found.\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 11, \""Determination of residues was found.\"", \""names_addresses_persons\"");\n section.addHintAnnotation(\""determination of residues\"", \""must_redact\"");\n section.addHintAnnotation(\""determination of total residues\"", \""must_redact\"");\n section.addHintAnnotation(\""livestock\"", \""must_redact\"");\n section.addHintAnnotation(\""live stock\"", \""must_redact\"");\n section.addHintAnnotation(\""tissue\"", \""must_redact\"");\n section.addHintAnnotation(\""tissues\"", \""must_redact\"");\n section.addHintAnnotation(\""liver\"", \""must_redact\"");\n section.addHintAnnotation(\""muscle\"", \""must_redact\"");\n section.addHintAnnotation(\""bovine\"", \""must_redact\"");\n section.addHintAnnotation(\""ruminant\"", \""must_redact\"");\n section.addHintAnnotation(\""ruminants\"", \""must_redact\"");\n end""",[CBI.15.0] -53,"""11: Redact Author cells in Tables with Author header (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && hasTableHeader(\""Author\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author\"", 11, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.10.0, CBI.11.0]" -54,"""12: Redact and add recommendation for et al. author""","""\n when\n Section(searchText.contains(\""et al\""))\n then\n\t\tsection.redactAndRecommendByRegEx(\""\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\"", false, 1, \""CBI_author\"", 12, \""Author found\"", \""names_addresses_persons\"");\n end""",[CBI.16.0] -55,"""12: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\"") || rowEquals(\""Vertebrate study Y/N\"", \""N\"") || rowEquals(\""Vertebrate study Y/N\"", \""No\"")))\n then\n section.redactCell(\""Author(s)\"", 12, \""CBI_author\"", true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.11.0]" -56,"""12: Recommend CTL/BL laboratory""","""\n when\n Section(searchText.contains(\""CT\"") || searchText.contains(\""BL\""))\n then\n section.addRecommendationByRegEx(\""((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\"", true, 0, \""CBI_address\"");\n end""",[SYN.1.0] -57,"""12: Redact if CTL/* or BL/* was found (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (searchText.contains(\""CTL/\"") || searchText.contains(\""BL/\"")))\n then\n section.addHintAnnotation(\""CTL\"", \""hint_only\"");\n section.addHintAnnotation(\""BL\"", \""hint_only\"");\n end""",[SYN.0.0] -58,"""13: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\"") || rowEquals(\""Vertebrate study Y/N\"", \""N\"") || rowEquals(\""Vertebrate study Y/N\"", \""No\"")))\n then\n section.redactCell(\""Author(s)\"", 13, \""CBI_author\"", true, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.10.0, CBI.11.0]" -59,"""14: Add recommendation for Addresses in Test Organism sections, 13: Add recommendation for Addresses in Test Organism sections""","""\n when\n Section(searchText.contains(\""Species:\"") && searchText.contains(\""Source:\""))\n then\n\t\tsection.recommendLineAfter(\""Source:\"", \""CBI_address\"");\n end""",[CBI.17.0] -60,"""13: Redact and add recommendation for et al. author""","""\n when\n Section(searchText.contains(\""et al\""))\n then\n\t\tsection.redactAndRecommendByRegEx(\""\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\"", false, 1, \""CBI_author\"", 13, \""Author found\"", \""names_addresses_persons\"");\n end""",[CBI.16.0] -61,"""13: Redact if CTL/* or BL/* was found (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (searchText.contains(\""CTL/\"") || searchText.contains(\""BL/\"")))\n then\n section.addRedaction(\""CTL\"", \""must_redact\"", 13, \""Laboratory for vertebrate studies found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"" );\n section.addRedaction(\""BL\"", \""must_redact\"", 13, \""Laboratory for vertebrate studies found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"" );\n end""",[SYN.0.0] -62,"""15: Add recommendation for Addresses in Test Animals sections, 14: Add recommendation for Addresses in Test Animals sections""","""\n when\n Section(searchText.contains(\""Species\"") && searchText.contains(\""Source\""))\n then\n\t\tsection.recommendLineAfter(\""Source\"", \""CBI_address\"");\n end""",[CBI.17.0] -63,"""14: Redact addresses that start with BL or CTL""","""\n when\n Section(searchText.contains(\""BL\"") || searchText.contains(\""CT\""))\n then\n section.addRecommendationByRegEx(\""((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\"", true, 0, \""CBI_address\"");\n end""",[SYN.1.0] -64,"""14: Redacted PII Personal Identification Information""","""\n when\n Section(matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 14, \""PII (Personal Identification Information) found\"", \""links_producer_applicant\"");\n end""","[PII.0.0, PII.0.1]" -65,"""14: Redact and add recommendation for et al. author (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""et al\""))\n then\n section.redactAndRecommendByRegEx(\""\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\"", false, 1, \""CBI_author\"", 14, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[CBI.16.0] -66,"""15: Redact Emails by RegEx""","""\n when\n Section(searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\"", true, 0, \""PII\"", 15, \""PII (Personal Identification Information) found\"", \""links_producer_applicant\"");\n end""","[PII.1.0, PII.1.1]" -67,"""15: Redact and add recommendation for et al. author (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""et al\""))\n then\n section.redactAndRecommendByRegEx(\""\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\"", false, 1, \""CBI_author\"", 15, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[CBI.16.0] -68,"""15: Redact and add recommendation for et al. author (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""et al\""))\n then\n section.redactAndRecommendByRegEx(\""\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\"", false, 1, \""CBI_author\"", 15, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.16.0] -69,"""16: Redacted PII Personal Identification Information""","""\n when\n Section(matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 16, \""PII (Personal Identification Information) found\"", \""links_producer_applicant\"");\n end""","[PII.0.0, PII.0.1]" -70,"""16: Redact and add recommendation for et al. author (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""et al\""))\n then\n section.redactAndRecommendByRegEx(\""\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\"", false, 1, \""CBI_author\"", 16, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.16.0] -71,"""16: Redact contact information""","""\n when\n Section(text.contains(\""Contact point:\"")\n || text.contains(\""Phone:\"")\n || text.contains(\""Fax:\"")\n || text.contains(\""Tel.:\"")\n || text.contains(\""Tel:\"")\n || text.contains(\""E-mail:\"")\n || text.contains(\""Email:\"")\n || text.contains(\""e-mail:\"")\n || text.contains(\""E-mail address:\"")\n || text.contains(\""Alternative contact:\"")\n || text.contains(\""Telephone number:\"")\n || text.contains(\""Telephone No:\"")\n || text.contains(\""Fax number:\"")\n || text.contains(\""Telephone:\"")\n || text.contains(\""European contact:\""))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 16, true, \""Contact information was found\"", \""links_producer_applicant\"");\n end""","[PII.4.0, PII.4.1, PII.6.0, PII.6.1]" -72,"""16: Add recommendation for Addresses in Test Organism sections, 17: Add recommendation for Addresses in Test Organism sections""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""Species:\"") && searchText.contains(\""Source:\""))\n then\n section.recommendLineAfter(\""Source:\"", \""CBI_address\"");\n end""",[CBI.17.0] -73,"""17: Redact Emails by RegEx""","""\n when\n Section(searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 17, \""PII (Personal Identification Information) found\"", \""links_producer_applicant\"");\n end""","[PII.1.0, PII.1.1]" -74,"""17: Add recommendation for Addresses in Test Animals sections, 18: Add recommendation for Addresses in Test Animals sections""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""Species\"") && searchText.contains(\""Source\""))\n then\n section.recommendLineAfter(\""Source\"", \""CBI_address\"");\n end""",[CBI.17.0] -75,"""17: Redact contact information if applicant is found""","""\n when\n Section(headlineContainsWord(\""applicant\"") || text.contains(\""Applicant\"") || headlineContainsWord(\""Primary contact\"") || headlineContainsWord(\""Alternative contact\"") || text.contains(\""Telephone number:\""))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 17, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n end""",[PII.7.0] -76,"""18: Do not redact Names and Addresses if Published Information found""","""\n when\n Section(matchesType(\""published_information\""))\n then\n section.redactNotAndReference(\""CBI_author\"",\""published_information\"", 18, \""Published Information found\"");\n section.redactNotAndReference(\""CBI_address\"",\""published_information\"", 18, \""Published Information found\"");\n end""",[CBI.7.0] -77,"""18: Redact contact information if Producer is found""","""\n when\n Section(text.toLowerCase().contains(\""producer of the plant protection\"") || text.toLowerCase().contains(\""producer of the active substance\"") || text.contains(\""Manufacturer of the active substance\"") || text.contains(\""Manufacturer:\"") || text.contains(\""Producer or producers of the active substance\""))\n then\n section.redactLineAfter(\""Contact:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 18, true, \""Producer was found\"", \""links_producer_applicant\"");\n end""",[PII.8.0] -78,"""18: Redact contact information""","""\n when\n Section(text.contains(\""Contact point:\"")\n || text.contains(\""Phone:\"")\n || text.contains(\""Fax:\"")\n || text.contains(\""Tel.:\"")\n || text.contains(\""Tel:\"")\n || text.contains(\""E-mail:\"")\n || text.contains(\""Email:\"")\n || text.contains(\""e-mail:\"")\n || text.contains(\""E-mail address:\"")\n || text.contains(\""Alternative contact:\"")\n || text.contains(\""Telephone number:\"")\n || text.contains(\""Telephone No:\"")\n || text.contains(\""Fax number:\"")\n || text.contains(\""Telephone:\"")\n || text.contains(\""Phone No.\"")\n || text.contains(\""European contact:\""))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 18, true, \""Contact information was found\"", \""links_producer_applicant\"");\n end""","[PII.4.0, PII.4.1, PII.6.0, PII.6.1]" -79,"""19: Redact contact information if applicant is found""","""\n when\n Section(headlineContainsWord(\""applicant\"") || text.contains(\""Applicant\"") || headlineContainsWord(\""Primary contact\"") || headlineContainsWord(\""Alternative contact\"") || text.contains(\""Telephone number:\""))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 19, true, \""Applicant information was found\"", \""links_producer_applicant\"");\n end""",[PII.7.0] -80,"""19: Redact AUTHOR(S)""","""\n when\n Section(searchText.contains(\""AUTHOR(S):\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 19, true, \""AUTHOR(S) was found\"", \""links_producer_applicant\"");\n end""","[PII.9.0, PII.9.1]" -81,"""19: Redacted PII Personal Identification Information (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 19, \""PII (Personal Identification Information) found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.0.0] -82,"""19: Redacted PII Personal Identification Information (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 19, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.0.0] -83,"""19: Do not redact Names and Addresses if Published Information found""","""\n when\n Section(matchesType(\""published_information\""))\n then\n section.redactNotAndReference(\""CBI_author\"",\""published_information\"", 19, \""Published Information found\"");\n section.redactNotAndReference(\""CBI_address\"",\""published_information\"", 19, \""Published Information found\"");\n end""",[CBI.7.0] -84,"""20: Redact contact information if Producer is found""","""\n when\n Section(text.toLowerCase().contains(\""producer of the plant protection\"") || text.toLowerCase().contains(\""producer of the active substance\"") || text.contains(\""Manufacturer of the active substance\"") || text.contains(\""Manufacturer:\"") || text.contains(\""Producer or producers of the active substance\""))\n then\n section.redactLineAfter(\""Contact:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 20, true, \""Producer was found\"", \""links_producer_applicant\"");\n end""",[PII.8.0] -85,"""20: Redact PERFORMING LABORATORY""","""\n when\n Section(searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""PII\"", 20, true, \""PERFORMING LABORATORY was found\"", \""links_producer_applicant\"");\n end""","[CBI.20.0, CBI.20.1]" -86,"""20: Redacted PII Personal Identification Information (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 20, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.0.1] -87,"""20: Redacted PII Personal Identification Information (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 20, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.0.1] -88,"""21: Redact Emails by RegEx (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\"", true, 0, \""PII\"", 21, \""PII (Personal Identification Information) found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.0] -89,"""21: Redact Emails by RegEx (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 21, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.0] -90,"""21: Redact On behalf of Sequani Ltd.:""","""\n when\n Section(searchText.contains(\""On behalf of Sequani Ltd.: Name Title\""))\n then\n section.redactBetween(\""On behalf of Sequani Ltd.: Name Title\"", \""On behalf of\"", \""PII\"", 21, false , \""PII (Personal Identification Information) found\"", \""links_producer_applicant\"");\n end""",[PII.11.0] -91,"""21: Redact AUTHOR(S)""","""\n when\n Section(searchText.contains(\""AUTHOR(S):\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 21, true, \""AUTHOR(S) was found\"", \""links_producer_applicant\"");\n end""","[PII.9.0, PII.9.1]" -92,"""22: Redact Emails by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\"", true, 0, \""PII\"", 22, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.1] -93,"""22: Redact Emails by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 22, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.1] -94,"""22: Redact PERFORMING LABORATORY""","""\n when\n Section(searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""PII\"", 22, true, \""PERFORMING LABORATORY was found\"", \""links_producer_applicant\"");\n end""","[CBI.20.0, CBI.20.1]" -95,"""22: Redact On behalf of Syngenta Ltd.:""","""\n when\n Section(searchText.contains(\""On behalf of Syngenta Ltd.: Name Title\""))\n then\n section.redactBetween(\""On behalf of Syngenta Ltd.: Name Title\"", \""Study dates\"", \""PII\"", 22, false , \""PII (Personal Identification Information) found\"", \""links_producer_applicant\"");\n end""",[PII.11.0] -96,"""23: Redact contact information (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (text.contains(\""Contact point:\"")\n || text.contains(\""Phone:\"")\n || text.contains(\""Fax:\"")\n || text.contains(\""Tel.:\"")\n || text.contains(\""Tel:\"")\n || text.contains(\""E-mail:\"")\n || text.contains(\""Email:\"")\n || text.contains(\""e-mail:\"")\n || text.contains(\""E-mail address:\"")\n || text.contains(\""Alternative contact:\"")\n || text.contains(\""Telephone number:\"")\n || text.contains(\""Telephone No:\"")\n || text.contains(\""Fax number:\"")\n || text.contains(\""Telephone:\"")\n || text.contains(\""Phone No.\"")\n || text.contains(\""European contact:\"")))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 23, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[PII.5.0, PII.6.0]" -97,"""23: Redact contact information (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (text.contains(\""Contact point:\"")\n || text.contains(\""Contact:\"")\n || text.contains(\""Alternative contact:\"")\n || (text.contains(\""No:\"") && text.contains(\""Fax\""))\n || (text.contains(\""Contact:\"") && text.contains(\""Tel.:\""))\n || text.contains(\""European contact:\"")\n ))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 23, true, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 23, true, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 23, true, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 23, true, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 23, true, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 23, true, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[PII.4.0, PII.6.0]" -98,"""24: Redact contact information (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (text.contains(\""Contact point:\"")\n || text.contains(\""Contact:\"")\n || text.contains(\""Alternative contact:\"")\n || (text.contains(\""No:\"") && text.contains(\""Fax\""))\n || (text.contains(\""Contact:\"") && text.contains(\""Tel.:\""))\n || text.contains(\""European contact:\"")\n ))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 24, true, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 24, true, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 24, true, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 24, true, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 24, true, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 24, true, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.5.1, PII.6.1]" -99,"""24: Redact contact information (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (text.contains(\""Contact point:\"")\n || text.contains(\""Phone:\"")\n || text.contains(\""Fax:\"")\n || text.contains(\""Tel.:\"")\n || text.contains(\""Tel:\"")\n || text.contains(\""E-mail:\"")\n || text.contains(\""Email:\"")\n || text.contains(\""e-mail:\"")\n || text.contains(\""E-mail address:\"")\n || text.contains(\""Alternative contact:\"")\n || text.contains(\""Telephone number:\"")\n || text.contains(\""Telephone No:\"")\n || text.contains(\""Fax number:\"")\n || text.contains(\""Telephone:\"")\n || text.contains(\""Phone No.\"")\n || text.contains(\""European contact:\"")))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 24, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.4.1, PII.6.1]" -100,"""25: Redact Phone and Fax by RegEx""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (\n text.contains(\""Contact\"")\n || text.contains(\""Telephone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Ph.\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Cell\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n ))\n then\n section.redactByRegEx(\""\\\\b(contact|telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\"", true, 2, \""PII\"", 25, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.0] -101,"""25: Redact Phone and Fax by RegEx (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (\n text.contains(\""Contact\"")\n || text.contains(\""Telephone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n ))\n then\n section.redactByRegEx(\""\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\"", true, 2, \""PII\"", 25, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.0] -102,"""25: Redact contact information if applicant is found (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (headlineContainsWord(\""applicant\"") || text.contains(\""Applicant\"") || headlineContainsWord(\""Primary contact\"") || headlineContainsWord(\""Alternative contact\"") || text.contains(\""Telephone number:\"")))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 25, true, \""Applicant information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.7.0] -103,"""33: Purity Hint, 34: Purity Hint, 50: Purity Hint, 25: Purity Hint, 39: Purity Hint""","""\n when\n Section(searchText.toLowerCase().contains(\""purity\""))\n then\n\t section.addHintAnnotationByRegEx(\""(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\"", true, 1, \""hint_only\"");\n end""",[ETC.0.0] -104,"""25: Redact Purity""","""\n when\n Section(searchText.contains(\""purity\""))\n then\n\t section.redactByRegEx(\""purity ?:? (([\\\\d\\\\.]+)( .{0,4}\\\\.)? ?%)\"", true, 1, \""purity\"", 17, \""Purity found\"", \""method_manufacture\"");\n end""",[ETC.1.0] -105,"""26: Redact Phone and Fax by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (\n text.contains(\""Contact\"")\n || text.contains(\""Telephone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Ph.\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Cell\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n ))\n then\n section.redactByRegEx(\""\\\\b(contact|telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\"", true, 2, \""PII\"", 26, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.1] -106,"""26: Redact contact information if applicant is found (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (headlineContainsWord(\""applicant\"") || text.contains(\""Applicant\"") || headlineContainsWord(\""Primary contact\"") || headlineContainsWord(\""Alternative contact\"") || text.contains(\""Telephone number:\"")))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 26, true, \""Applicant information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.7.0] -107,"""26: Redact signatures""","""\n when\n Section(matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 26, \""Signature found\"", \""names_addresses_persons\"");\n end""",[ETC.2.0] -108,"""26: Redact Phone and Fax by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (\n text.contains(\""Contact\"")\n || text.contains(\""Telephone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n ))\n then\n section.redactByRegEx(\""\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\"", true, 2, \""PII\"", 26, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.1] -109,"""27: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""COMPLETION DATE:\"")\n && !searchText.contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -110,"""27: Redact Logos""","""\n when\n Section(matchesImageType(\""logo\""))\n then\n section.redactImage(\""logo\"", 27, \""Logo found\"", \""names_addresses_persons\"");\n end""",[ETC.3.0] -111,"""27: Redact contact information if Producer is found (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (text.toLowerCase().contains(\""producer of the plant protection\"") || text.toLowerCase().contains(\""producer of the active substance\"") || text.contains(\""Manufacturer of the active substance\"") || text.contains(\""Manufacturer:\"") || text.contains(\""Producer or producers of the active substance\"")))\n then\n section.redactLineAfter(\""Contact:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 27, true, \""Producer was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.8.0] -112,"""28: Redact contact information if Producer is found (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (text.toLowerCase().contains(\""producer of the plant protection\"") || text.toLowerCase().contains(\""producer of the active substance\"") || text.contains(\""Manufacturer of the active substance\"") || text.contains(\""Manufacturer:\"") || text.contains(\""Producer or producers of the active substance\"")))\n then\n section.redactLineAfter(\""Contact:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 28, true, \""Producer was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.8.0] -113,"""28: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""COMPLETION DATE:\"")\n && !searchText.contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 28, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -114,"""28: Redact dossier dictionary match""","""\n when\n Section(matchesType(\""dossier_redaction\""))\n then\n section.redact(\""dossier_redaction\"", 28, \""Specification of impurity found\"", \""specification_impurity_active_substance\"");\n end""","[ETC.4.0, ETC.4.1, ETC.4.2]" -115,"""29: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""COMPLETION DATE:\"") && !searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -116,"""34: Ignore dossier_redaction entries if confidentiality is not 'confidential', 51: Ignore dossier_redaction entries if confidential, 29: Ignore dossier_redaction unless confidential, 40: Ignore dossier_redaction entries if confidential""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Confidentiality\"",\""confidential\"") && matchesType(\""dossier_redaction\""));\n then\n section.ignore(\""dossier_redaction\"");\n end""",[ETC.5.0] -117,"""29: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""PII\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -118,"""30: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""PII\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -119,"""30: Redacted PII Personal Identification Information (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 30, \""PII (Personal Identification Information) found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.0.0] -120,"""30: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""COMPLETION DATE:\"") && !searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -121,"""31: Redacted PII Personal Identification Information (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 31, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.0.1] -122,"""31: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""PII\"", 31, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -123,"""31: Redact PERFORMING LABORATORY (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""PERFORMING LABORATORY:\"")\n )\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""CBI_address\"", 31, true, \""PERFORMING LABORATORY was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactNot(\""CBI_address\"", 31, \""Performing laboratory found for non vertebrate study\"");\n end""",[CBI.20.0] -124,"""32: Redact PERFORMING LABORATORY (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""CBI_address\"", 32, true, \""PERFORMING LABORATORY was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.20.1] -125,"""32: Redact Emails by RegEx (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 32, \""PII (Personal Identification Information) found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.0] -126,"""32: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""PII\"", 32, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -127,"""33: Redact Emails by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 33, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.1] -128,"""33: Redact study director abbreviation""","""\n when\n Section((searchText.contains(\""KATH\"") || searchText.contains(\""BECH\"") || searchText.contains(\""KML\"")))\n then\n section.redactWordPartByRegEx(\""((KATH)|(BECH)|(KML)) ?(\\\\d{4})\"", true, 0, 1, \""PII\"", 34, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.10.0] -129,"""33: Redact PERFORMING LABORATORY (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""CBI_address\"", 33, true, \""PERFORMING LABORATORY was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactNot(\""CBI_address\"", 33, \""Performing laboratory found for non vertebrate study\"");\n end""",[CBI.20.0] -130,"""34: Redact PERFORMING LABORATORY (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""CBI_address\"", 34, true, \""PERFORMING LABORATORY was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.20.1] -131,"""34: Redact telephone numbers by RegEx (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && containsRegEx(\""[+]\\\\d{1,}\"", true))\n then\n section.redactByRegEx(\""((([+]\\\\d{1,3} (\\\\d{7,12})\\\\b)|([+]\\\\d{1,3}(\\\\d{3,12})\\\\b|[+]\\\\d{1,3}([ -]\\\\(?\\\\d{1,6}\\\\)?){2,4})|[+]\\\\d{1,3} ?((\\\\d{2,6}\\\\)?)([ -]\\\\d{2,6}){1,4}))(-\\\\d{1,3})?\\\\b)\"", true, 1, \""PII\"", 34, \""PII (Personal Identification Information) found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.3.0] -132,"""35: Redact telephone numbers by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && containsRegEx(\""[+]\\\\d{1,}\"", true))\n then\n section.redactByRegEx(\""((([+]\\\\d{1,3} (\\\\d{7,12})\\\\b)|([+]\\\\d{1,3}(\\\\d{3,12})\\\\b|[+]\\\\d{1,3}([ -]\\\\(?\\\\d{1,6}\\\\)?){2,4})|[+]\\\\d{1,3} ?((\\\\d{2,6}\\\\)?)([ -]\\\\d{2,6}){1,4}))(-\\\\d{1,3})?\\\\b)\"", true, 1, \""PII\"", 35, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.3.1] -133,"""35: Redact signatures (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 35, \""Signature found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[ETC.2.0] -134,"""36: Redact signatures (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 36, \""Signature found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[ETC.2.0] -135,"""37: Redact contact information (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (text.contains(\""Contact point:\"")\n || text.contains(\""Phone:\"")\n || text.contains(\""Fax:\"")\n || text.contains(\""Tel.:\"")\n || text.contains(\""Tel:\"")\n || text.contains(\""E-mail:\"")\n || text.contains(\""Email:\"")\n || text.contains(\""e-mail:\"")\n || text.contains(\""E-mail address:\"")\n || text.contains(\""Alternative contact:\"")\n || text.contains(\""Telephone number:\"")\n || text.contains(\""Telephone No:\"")\n || text.contains(\""Fax number:\"")\n || text.contains(\""Telephone:\"")\n || text.contains(\""Phone No.\"")\n || text.contains(\""European contact:\"")))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 37, true, \""Contact information was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[PII.4.0, PII.6.0]" -136,"""38: Redact contact information (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (text.contains(\""Contact point:\"")\n || text.contains(\""Phone:\"")\n || text.contains(\""Fax:\"")\n || text.contains(\""Tel.:\"")\n || text.contains(\""Tel:\"")\n || text.contains(\""E-mail:\"")\n || text.contains(\""Email:\"")\n || text.contains(\""e-mail:\"")\n || text.contains(\""E-mail address:\"")\n || text.contains(\""Alternative contact:\"")\n || text.contains(\""Telephone number:\"")\n || text.contains(\""Telephone No:\"")\n || text.contains(\""Fax number:\"")\n || text.contains(\""Telephone:\"")\n || text.contains(\""Phone No.\"")\n || text.contains(\""European contact:\"")))\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 38, true, \""Contact information was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.4.1, PII.6.1]" -137,"""39: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""COMPLETION DATE:\"") && !searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 39, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -138,"""40: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""COMPLETION DATE:\"") && !searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 40, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -139,"""41: Redact signatures (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 41, \""Signature found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[ETC.2.0] -140,"""41: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""PII\"", 41, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -141,"""42: Redact signatures (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 42, \""Signature found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[ETC.2.0] -142,"""42: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""PII\"", 42, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -143,"""43: Redact PERFORMING LABORATORY (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""CBI_address\"", 43, true, \""PERFORMING LABORATORY was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactNot(\""CBI_address\"", 43, \""Performing laboratory found for non vertebrate study\"");\n end""",[CBI.20.0] -144,"""43: Redact Logos (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesImageType(\""logo\""))\n then\n section.redactImage(\""logo\"", 43, \""Logo found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[ETC.3.0] -145,"""44: Redact PERFORMING LABORATORY (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""CBI_address\"", 44, true, \""PERFORMING LABORATORY was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.20.1] -146,"""52: Redact signatures (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 52, \""Signature found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[ETC.2.0] -147,"""53: Redact signatures (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 53, \""Signature found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[ETC.2.0] -148,"""54: Redact Logos (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesImageType(\""logo\""))\n then\n section.redactImage(\""logo\"", 54, \""Logo found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[ETC.3.0] -149,"""0: Expand CBI Authors with firstname initials""","""\n when\n Section(matchesType(\""CBI_author\""))\n then\n section.expandByRegEx(\""CBI_author\"", \""(,? [A-Z]\\\\.?( ?[A-Z]\\\\.?)?( ?[A-Z]\\\\.?)?\\\\b\\\\.?)\"", false, 1);\n end""",[CBI.18.0] -150,"""0: Expand CBI Authors with firstname initials""","""\n when\n Section(matchesType(\""CBI_author\""))\n then\n section.expandByRegEx(\""CBI_author\"", \""(,? [A-Z]\\\\.?( ?[A-Z]\\\\.?)?( ?[A-Z]\\\\.?)?\\\\b\\\\.?)\"", false, 1, \""[^\\\\s]+\"");\n end""",[CBI.18.0] -151,"""0: Expand CBI_author and PII matches with salutation prefix""","""\n when\n Section((matchesType(\""CBI_author\"") || matchesType(\""PII\"")) && (\n searchText.contains(\""Mr\"")\n || searchText.contains(\""Mrs\"")\n || searchText.contains(\""Ms\"")\n || searchText.contains(\""Miss\"")\n || searchText.contains(\""Sir\"")\n || searchText.contains(\""Madam\"")\n || searchText.contains(\""Madame\"")\n || searchText.contains(\""Mme\"")\n ))\n then\n section.expandByPrefixRegEx(\""CBI_author\"", \""\\\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\\\s?\\\\.?\\\\s*\"", false, 0);\n section.expandByPrefixRegEx(\""PII\"", \""\\\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\\\s?\\\\.?\\\\s*\"", false, 0);\n end""","[CBI.19.0, PII.12.0]" -152,"""102: Guidelines FileAttributes""","""\n when\n Section((text.contains(\""DATA REQUIREMENT(S):\"") || text.contains(\""TEST GUIDELINE(S):\"")) && (text.contains(\""OECD\"") || text.contains(\""EPA\"") || text.contains(\""OPPTS\"")))\n then\n section.addFileAttribute(\""OECD Number\"", \""OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\"", false, 0);\n end""",[ETC.7.0] -153,"""28: Redact Logos""","""\n when\n Section(matchesImageType(\""logo\""))\n then\n section.redactImage(\""logo\"", 28, \""Logo found\"", \""names_addresses_persons\"");\n end""",[ETC.3.0] -154,"""8: Redact Author cells in Tables with Author header (Non vertebrate study)""","""\n when\n Section(hasTableHeader(\""h5.1\""))\n then\n section.redactCell(\""h5.1\"", 8, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.11.0]" -155,"""30: Ignore dossier_redactions if confidential""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Confidentiality\"",\""confidential\"") && matchesType(\""dossier_redactions\""));\n then\n section.ignore(\""dossier_redactions\"");\n end""",[ETC.5.0] -156,"""27: Redact formula""","""\n when\n Section(matchesImageType(\""formula\""))\n then\n section.redactImage(\""formula\"", 27, \""Formula found\"", \""names_addresses_persons\"");\n end""",[ETC.8.0] -157,"""5: Do not redact Names and Addresses if no redaction Indicator is contained""","""\n when\n Section(matchesType(\""vertebrate\""), matchesType(\""published_information\""))\n then\n section.redactNotAndReference(\""CBI_author\"",\""published_information\"", 5, \""Vertebrate and Published Information found\"");\n section.redactNotAndReference(\""CBI_address\"",\""published_information\"", 5, \""Vertebrate and Published Information found\"");\n end""",[CBI.4.0] -158,"""29: Redact Dossier Redactions""","""\n when\n Section(matchesType(\""dossier_redactions\""))\n then\n section.redact(\""dossier_redactions\"", 29, \""Dossier Redaction found\"", \""Article 39(1)(2) of Regulation (EC) No 178/2002\"");\n end""","[ETC.4.0, ETC.4.1, ETC.4.2]" -159,"""10: Redact determination of residues""","""\n when\n Section((\n searchText.toLowerCase.contains(\""determination of residues\"") ||\n searchText.toLowerCase.contains(\""determination of total residues\"")\n ) && (\n searchText.toLowerCase.contains(\""livestock\"") ||\n searchText.toLowerCase.contains(\""live stock\"") ||\n searchText.toLowerCase.contains(\""tissue\"") ||\n searchText.toLowerCase.contains(\""tissues\"") ||\n searchText.toLowerCase.contains(\""liver\"") ||\n searchText.toLowerCase.contains(\""muscle\"") ||\n searchText.toLowerCase.contains(\""bovine\"") ||\n searchText.toLowerCase.contains(\""ruminant\"") ||\n searchText.toLowerCase.contains(\""ruminants\"")\n ))\n then\n section.redact(\""CBI_author\"", 10, \""Determination of residues was found.\"", \""names_addresses_persons\"");\n section.redact(\""CBI_address\"", 10, \""Determination of residues was found.\"", \""names_addresses_persons\"");\n section.addHintAnnotation(\""determination of residues\"", \""must_redact\"");\n section.addHintAnnotation(\""livestock\"", \""must_redact\"");\n section.addHintAnnotation(\""live stock\"", \""must_redact\"");\n section.addHintAnnotation(\""tissue\"", \""must_redact\"");\n section.addHintAnnotation(\""tissues\"", \""must_redact\"");\n section.addHintAnnotation(\""liver\"", \""must_redact\"");\n section.addHintAnnotation(\""muscle\"", \""must_redact\"");\n section.addHintAnnotation(\""bovine\"", \""must_redact\"");\n section.addHintAnnotation(\""ruminant\"", \""must_redact\"");\n section.addHintAnnotation(\""ruminants\"", \""must_redact\"");\n end""",[CBI.15.0] -160,"""19: Redact AUTHOR(S)""","""\n when\n Section(searchText.contains(\""AUTHOR(S):\"") && fileAttributeByPlaceholderEquals(\""{fileattributes.vertebrateStudy}\"", \""true\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 19, true, \""AUTHOR(S) was found\"", \""links_producer_applicant\"");\n end""",[PII.9.1] -161,"""101: Redact CAS numbers""","""\n when\n Section(hasTableHeader(\""Sample #\""))\n then\n section.redactCell(\""Sample #\"", 8, \""PII\"", true, \""Redacted because row is a vertebrate study\"", \""names_addresses_persons\"");\n end""",[ETC.6.0] -162,"""21: Redact Emails by RegEx""","""\n when\n Section(searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 21, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.1.0, PII.1.1]" -163,"""32: Redact signatures""","""\n when\n Section(matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 32, \""Signature found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[ETC.2.0, ETC.2.1]" -164,"""11: Do not redact Names and Addresses if Published Information found""","""\n when\n Section(matchesType(\""published_information\""))\n then\n section.redactNotAndReference(\""CBI_author\"",\""published_information\"", 11, \""Published Information found\"");\n section.redactNotAndReference(\""CBI_address\"",\""published_information\"", 11, \""Published Information found\"");\n end""","[CBI.7.0, CBI.7.1]" -165,"""9: Add recommendation for Addresses in Test Organism sections""","""\n when\n Section(searchText.contains(\""Species:\"") && searchText.contains(\""Source:\""))\n then\n section.recommendLineAfter(\""Source:\"", \""CBI_address\"");\n end""",[CBI.17.1] -166,"""5: Redact Author cells in Tables with Author header""","""\n when\n Section(hasTableHeader(\""Author\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author\"", 5, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.1, CBI.10.1]" -167,"""21: Redact Phone and Fax by RegEx""","""\n when\n Section(\n text.contains(\""Telephone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Ph.\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Cell\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n )\n then\n section.redactByRegEx(\""\\\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\\\s]{0,3}((\\\\(?\\\\+?[0-9])(\\\\(?[0-9\\\\/.\\\\-\\\\s]+\\\\)?)*([0-9]+\\\\)?))\\\\b\"", true, 2, \""PII\"", 23, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.2.0, PII.2.1]" -168,"""2: Redact CBI Address""","""\n when\n Section(matchesType(\""CBI_address\""))\n then\n section.redact(\""CBI_address\"", 4, \""Address found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.1.1] -169,"""25: Redact AUTHOR(S)""","""\n when\n Section(searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""COMPLETION DATE:\"")\n && !searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 25, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.9.0, PII.9.1]" -170,"""6: Redact and recommand Authors in Tables with Vertebrate study Y/N header""","""\n when\n Section(rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\"") || rowEquals(\""Vertebrate study Y/N\"", \""N\"") || rowEquals(\""Vertebrate study Y/N\"", \""No\""))\n then\n section.redactCell(\""Author(s)\"", 6, \""CBI_author\"", true, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.9.1, CBI.10.0, CBI.10.1, CBI.11.0, CBI.12.0, CBI.12.1]" -171,"""22: Redact contact information""","""\n when\n Section(text.contains(\""Contact point:\"")\n || text.contains(\""Phone:\"")\n || text.contains(\""Fax:\"")\n || text.contains(\""Tel.:\"")\n || text.contains(\""Tel:\"")\n || text.contains(\""E-mail:\"")\n || text.contains(\""Email:\"")\n || text.contains(\""e-mail:\"")\n || text.contains(\""E-mail address:\"")\n || text.contains(\""Contact:\"")\n || text.contains(\""Alternative contact:\"")\n || text.contains(\""Telephone number:\"")\n || text.contains(\""Telephone No:\"")\n || text.contains(\""Fax number:\"")\n || text.contains(\""Telephone:\"")\n || text.contains(\""Phone No.\"")\n || (text.contains(\""No:\"") && text.contains(\""Fax\""))\n || (text.contains(\""Contact:\"") && text.contains(\""Tel.:\""))\n || text.contains(\""European contact:\"")\n )\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 22, true, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.4.0, PII.4.1, PII.5.0, PII.5.1, PII.6.0, PII.6.1, PII.7.0, PII.7.1, PII.8.0, PII.8.1]" -172,"""20: Redacted PII Personal Identification Information""","""\n when\n Section(matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 20, \""PII (Personal Identification Information) found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.0.0, PII.0.1]" -173,"""1: Redact CBI Authors""","""\n when\n Section(matchesType(\""CBI_author\""))\n then\n section.redact(\""CBI_author\"", 1, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.0.0, CBI.0.1]" -174,"""4: Redact Author(s) cells in Tables with Author(s) header""","""\n when\n Section(hasTableHeader(\""Author(s)\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author(s)\"", 4, \""CBI_author\"", false, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.9.0, CBI.10.0]" -175,"""8: Redact and add recommendation for et al. author""","""\n when\n Section(searchText.contains(\""et al\""))\n then\n section.redactAndRecommendByRegEx(\""\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\"", false, 1, \""CBI_author\"", 15, \""Author found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.16.0, CBI.16.1]" -176,"""26: Redact AUTHOR(S)""","""\n when\n Section(searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""PII\"", 26, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[PII.9.0, PII.9.1]" -177,"""27: Redact PERFORMING LABORATORY""","""\n when\n Section(searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""CBI_address\"", 27, true, \""PERFORMING LABORATORY was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[CBI.20.0, CBI.20.1]" -178,"""10: Add recommendation for Addresses in Test Animals sections""","""\n when\n Section(searchText.contains(\""Species\"") && searchText.contains(\""Source\""))\n then\n section.recommendLineAfter(\""Source\"", \""CBI_address\"");\n end""","[CBI.17.0, CBI.17.1]" -179,"""33: Redact Logos""","""\n when\n Section(matchesImageType(\""logo\""))\n then\n section.redactImage(\""logo\"", 33, \""Logo found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""","[ETC.3.0, ETC.3.1]" -180,"""26: Redact Phone and Fax by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (\n text.contains(\""Contact\"")\n || text.contains(\""Telephone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Ph.\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n ))\n then\n section.redactByRegEx(\""\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\"", true, 2, \""PII\"", 26, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.1] -181,"""25: Redact Phone and Fax by RegEx (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (\n text.contains(\""Contact\"")\n || text.contains(\""Telephone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Ph.\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n ))\n then\n section.redactByRegEx(\""\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\"", true, 2, \""PII\"", 25, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.0] -182,"""10: Add recommendation for Addresses in Test Animals sections""","""\n when\n Section(searchText.contains(\""Species\"") && searchText.contains(\""Source\""))\n then\n\t\tsection.recommendLineAfter(\""Source\"", \""PII\"");\n end""","[CBI.17.0, CBI.17.1]" -183,"""0: Combine address parts from AI to addresses as PII (org is mandatory)""","""\n when\n Section(aiMatchesType(\""ORG\""))\n then\n section.combineAiTypes(\""ORG\"", \""STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\"", 20, \""PII\"", 3, false);\n end""",[AI.1.0] -184,"""0: Combine address parts from AI to addresses as PII (street is mandatory)""","""\n when\n Section(aiMatchesType(\""STREET\""))\n then\n section.combineAiTypes(\""STREET\"", \""ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\"", 20, \""PII\"", 3, false);\n end""",[AI.1.0] -185,"""18: Redact AUTHOR(S)""","""\n when\n Section(searchText.contains(\""AUTHOR(S):\"") && searchText.contains(\""STUDY COMPLETION DATE:\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""PII\"", 18, true, \""AUTHOR(S) was found\"", \""personal_data_geolocation\"");\n end""","[PII.9.0, PII.9.1]" -186,"""0: Combine address parts from AI to addresses as PII (city is mandatory)""","""\n when\n Section(aiMatchesType(\""CITY\""))\n then\n section.combineAiTypes(\""CITY\"", \""ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\"", 20, \""PII\"", 3, false);\n end""",[AI.1.0] -187,"""24: Redact signatures""","""\n when\n Section(matchesImageType(\""signature\""))\n then\n section.redactImage(\""signature\"", 24, \""Signature found\"", \""personal_data_geolocation\"");\n end""","[ETC.2.0, ETC.2.1]" -188,"""13: Redact Emails by RegEx""","""\n when\n Section(searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 13, \""PII (Personal Identification Information) found\"", \""personal_data_geolocation\"");\n end""","[PII.1.0, PII.1.1]" -189,"""4: Redact Author(s) cells in Tables with Author(s) header""","""\n when\n Section(hasTableHeader(\""Author(s)\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author(s)\"", 4, \""PII\"", false, \""Author found\"", \""personal_data_geolocation\"");\n end""","[CBI.9.0, CBI.10.0, CBI.11.0]" -190,"""25: Redact Logos""","""\n when\n Section(matchesImageType(\""logo\""))\n then\n section.redactImage(\""logo\"", 25, \""Logo found\"", \""personal_data_geolocation\"");\n end""","[ETC.3.0, ETC.3.1]" -191,"""11: Do not redact Names and Addresses if Published Information found""","""\n when\n Section(matchesType(\""published_information\""))\n then\n section.redactNot(\""PII\"", 11, \""Published Information found\"");\n section.redactNot(\""PII\"", 11, \""Published Information found\"");\n end""","[CBI.7.0, CBI.7.1, CBI.6.0, CBI.6.1]" -192,"""2: Do not redact genitive PIIs""","""\n when\n Section(matchesType(\""PII\""))\n then\n section.expandToFalsePositiveByRegEx(\""PII\"", \""['’’'ʼˈ´`‘′ʻ’']s\"", false, 0);\n end""",[CBI.2.0] -193,"""5: Redact Author cells in Tables with Author header""","""\n when\n Section(hasTableHeader(\""Author\"") && !hasTableHeader(\""Vertebrate study Y/N\""))\n then\n section.redactCell(\""Author\"", 5, \""PII\"", false, \""Author found\"", \""personal_data_geolocation\"");\n end""","[CBI.9.1, CBI.10.1, CBI.12.0]" -194,"""19: Redact PERFORMING LABORATORY""","""\n when\n Section(searchText.contains(\""PERFORMING LABORATORY:\""))\n then\n section.redactBetween(\""PERFORMING LABORATORY:\"", \""LABORATORY PROJECT ID:\"", \""PII\"", 19, true, \""PERFORMING LABORATORY was found\"", \""personal_data_geolocation\"");\n end""","[CBI.20.0, CBI.20.1]" -195,"""0: Recommend authors from AI as PII""","""\n when\n Section(aiMatchesType(\""CBI_author\""))\n then\n section.addAiEntities(\""CBI_author\"", \""PII\"");\n end""",[AI.3.0] -196,"""9: Add recommendation for Addresses in Test Organism sections""","""\n when\n Section(searchText.contains(\""Species:\"") && searchText.contains(\""Source:\""))\n then\n\t\tsection.recommendLineAfter(\""Source:\"", \""PII\"");\n end""",[CBI.17.1] -197,"""1: Redacted PII Personal Identification Information""","""\n when\n Section(matchesType(\""PII\""))\n then\n section.redact(\""PII\"", 1, \""Personal information found\"", \""personal_data_geolocation\"");\n end""","[PII.0.0, PII.0.1]" -198,"""8: Redact and add recommendation for et al. author""","""\n when\n Section(searchText.contains(\""et al\""))\n then\n\t\tsection.redactAndRecommendByRegEx(\""\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\"", false, 1, \""PII\"", 8, \""Author found\"", \""personal_data_geolocation\"");\n end""","[CBI.16.0, CBI.16.1]" -199,"""14: Redact contact information""","""\n when\n Section(text.contains(\""Contact point:\"")\n || text.contains(\""Phone:\"")\n || text.contains(\""Fax:\"")\n || text.contains(\""Tel.:\"")\n || text.contains(\""Tel:\"")\n || text.contains(\""E-mail:\"")\n || text.contains(\""Email:\"")\n || text.contains(\""e-mail:\"")\n || text.contains(\""E-mail address:\"")\n || text.contains(\""Contact:\"")\n || text.contains(\""Alternative contact:\"")\n || text.contains(\""Telephone number:\"")\n || text.contains(\""Telephone No:\"")\n || text.contains(\""Fax number:\"")\n || text.contains(\""Telephone:\"")\n || text.contains(\""Phone No.\"")\n || (text.contains(\""No:\"") && text.contains(\""Fax\""))\n || (text.contains(\""Contact:\"") && text.contains(\""Tel.:\""))\n || text.contains(\""European contact:\"")\n )\n then\n section.redactLineAfter(\""Contact point:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Phone:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Fax:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Tel.:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Tel:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""E-mail:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Email:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""e-mail:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""E-mail address:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Contact:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Alternative contact:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Telephone number:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Telephone No:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Fax number:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Telephone:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""Phone No.\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactBetween(\""No:\"", \""Fax\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactBetween(\""Contact:\"", \""Tel.:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n section.redactLineAfter(\""European contact:\"", \""PII\"", 14, true, \""Personal information found\"", \""personal_data_geolocation\"");\n end""","[PII.4.0, PII.4.1, PII.6.0, PII.6.1]" -200,"""17: Redact AUTHOR(S)""","""\n when\n Section(searchText.contains(\""AUTHOR(S):\""))\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""PII\"", 17, true, \""AUTHOR(S) was found\"", \""personal_data_geolocation\"");\n end""","[PII.9.0, PII.9.1]" -201,"""6: Redact and recommand Authors in Tables with Vertebrate study Y/N header""","""\n when\n Section(rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\"") || rowEquals(\""Vertebrate study Y/N\"", \""N\"") || rowEquals(\""Vertebrate study Y/N\"", \""No\""))\n then\n section.redactCell(\""Author(s)\"", 6, \""PII\"", false, \""Author found\"", \""personal_data_geolocation\"");\n end""","[CBI.11.0, CBI.12.2, CBI.12.1]" -202,"""27: Redact Logos""","""\n °when\n Section(matchesImageType(\""logo\""))\n then\n //section.redactImage(\""logo\"", 27, \""Logo found\"", \""names_addresses_persons\"");\n section.redactNotImage(\""logo\"", 27, \""No Logos in preGFL documents\"");\n end""","[ETC.3.0, ETC.3.1]" -203,"""27: Redact Logos""","""\n when\n Section(matchesImageType(\""logo\""))\n then\n //section.redactImage(\""logo\"", 27, \""Logo found\"", \""names_addresses_persons\"");\n section.redactNotImage(\""logo\"", 27, \""No Logos in preGFL documents\"");\n end""","[ETC.3.0, ETC.3.1]" -204,"""10a: Redact Addresses in Reference Tables for vertebrate studies in non-vertebrate documents""","""\n when\n Section(hasTableHeader(\""Vertebrate study Y/N\"") && (rowEquals(\""Vertebrate study Y/N\"", \""Y\"") || rowEquals(\""Vertebrate study Y/N\"", \""Yes\"")))\n then\n section.redact(\""CBI_address\"", 10, \""Redacted because row is a vertebrate study\"", \""names_addresses_persons\"");\n end""",[CBI.22.0] -205,"""27a: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toUpperCase().contains(\""AUTHOR(S)\"")\n && !searchText.toUpperCase().contains(\""AUTHOR(S):\"")\n && searchText.toUpperCase().contains(\""COMPLETION DATE\"")\n && !searchText.toUpperCase().contains(\""STUDY COMPLETION DATE\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S)\"", \""COMPLETION DATE\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\"", \""Completion Date\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""AUTHOR(S)\\n\"", \""COMPLETION DATE\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\\n\"", \""Completion Date\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -206,"""29b: Redact short Authors section""","""\n when\n Section(\n !fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toLowerCase().contains(\""author(s)\"")\n && searchText.length() < 50\n && sectionNumber <= 20\n && !aiMatchesType(\""CBI_Author\"")\n )\n then\n section.redactByRegEx(\""(?<=author\\\\(?s\\\\)?\\\\s\\\\n?)([\\\\p{Lu}\\\\p{L} ]{5,15}(,|\\\\n)?){1,3}\"",true,0,\""CBI_author\"",29,\""AUTHOR(S) was found\"",\""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[CBI.21.0] -207,"""28a: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toUpperCase().contains(\""AUTHOR(S)\"")\n && !searchText.toUpperCase().contains(\""AUTHOR(S):\"")\n && searchText.toUpperCase().contains(\""COMPLETION DATE\"")\n && !searchText.toUpperCase().contains(\""STUDY COMPLETION DATE\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S)\"", \""COMPLETION DATE\"", \""CBI_author\"", 28, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\"", \""Completion Date\"", \""CBI_author\"", 28, true, \""Author(s) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""AUTHOR(S)\\n\"", \""COMPLETION DATE\"", \""CBI_author\"", 28, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\\n\"", \""Completion Date\"", \""CBI_author\"", 28, true, \""Author(s) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -208,"""65: Redact Skipped Impurities""","""\n when\n Section((fileAttributeByLabelEqualsIgnoreCase(\""Redact Skipped Impurities\"",\""Yes\"") || fileAttributeByLabelEqualsIgnoreCase(\""Redact Skipped Impurities\"",\""Y\"")) && matchesType(\""skipped_impurities\""))\n then\n section.redact(\""skipped_impurities\"", 65, \""Occasional Impurity found\"", \""specification_impurity\"");\n end""",[ETC.9.0] -209,"""19c: Recommend first line in table cell with name and address of owner""","""\n when\n Section(searchText.toLowerCase().contains(\""trial site\"") && hasTableHeader(\""Name and Address of Owner / Tenant\""))\n then\n section.redactCell(\""Name and Address of Owner / Tenant\"",19,\""PII\"",true,\""Trial Site owner and address found\"",\""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[ETC.11.0] -210,"""67: Redact Product Composition Information""","""\n when\n Section(matchesType(\""product_composition\""))\n then\n section.redact(\""product_composition\"",67, \""Product Composition Information found\"",\""composition_plant_protection_product\"");\n end""",[ETC.10.0] -211,"""25: Redact Phone and Fax by RegEx (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (\n text.contains(\""Contact\"")\n || text.contains(\""Telephone\"")\n || text.contains(\""Téléphone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n ))\n then\n section.redactByRegEx(\""\\\\b(contact|t\\\\p{Ll}l\\\\p{Ll}phone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s\\\\.]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(O][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.O]{4,100}\\\\d)\\\\b\"", true, 2, \""PII\"", 25, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.0] -212,"""19a: recommend title prefixed words as PII""","""\n when\n Section(\n searchText.contains(\""Dr \"")\n || searchText.contains(\""PD Dr \"")\n || searchText.contains(\""Prof. Dr \"")\n || searchText.contains(\""Dr. med. vet \"")\n || searchText.contains(\""Dr. rer. nat \"")\n || searchText.contains(\""PhD \"")\n || searchText.contains(\""BSc \"")\n || searchText.contains(\""(FH) \"")\n || searchText.contains(\""Mr \"")\n || searchText.contains(\""Mrs \"")\n || searchText.contains(\""Ms \"")\n || searchText.contains(\""Miss \"")\n || searchText.contains(\""Dr.\"")\n || searchText.contains(\""PD Dr.\"")\n || searchText.contains(\""Prof. Dr.\"")\n || searchText.contains(\""Dr. med. vet.\"")\n || searchText.contains(\""Dr. rer. nat.\"")\n || searchText.contains(\""PhD.\"")\n || searchText.contains(\""BSc.\"")\n || searchText.contains(\""(FH).\"")\n || searchText.contains(\""Mr.\"")\n || searchText.contains(\""Mrs.\"")\n || searchText.contains(\""Ms.\"")\n || searchText.contains(\""Miss.\"")\n )\n then\n section.addRecommendationByRegEx(\""((Dr|PD Dr|Prof. Dr|Dr. med. vet|Dr. rer. nat|PhD|BSc|\\\\(FH\\\\)|Mr|Mrs|Ms|Miss)[.\\\\s]{1,2})([\\\\p{Lu}][\\\\p{L}\\\\-.]{1,20}\\\\s[\\\\p{Lu}][\\\\p{L}\\\\-.]{1,20})\"", false, 3, \""PII\"");\n end""",[PII.14.0] -213,"""0: Combine address parts from ai to CBI_address (city is mandatory)""","""\n when\n Section(aiMatchesType(\""CITY\""))\n then\n section.combineAiTypes(\""CITY\"", \""ORG,DEPARTMENT,STREET,POSTAL,COUNTRY,CARDINAL,STATE\"", 20, \""CBI_address\"", 3, false);\n end""",[AI.1.0] -214,"""29: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toUpperCase().contains(\""AUTHOR(S):\"")\n && searchText.toUpperCase().contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\"", \""Study Completion Date:\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\"", \""Study completion Date:\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""AUTHOR(S):\\n\"", \""STUDY COMPLETION DATE:\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\\n\"", \""Study Completion Date:\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\\n\"", \""Study completion Date:\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -215,"""22: Redact Emails by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@\\\\s?[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 22, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.1] -216,"""27: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toUpperCase().contains(\""AUTHOR(S):\"")\n && searchText.toUpperCase().contains(\""COMPLETION DATE:\"")\n && !searchText.toUpperCase().contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\"", \""Completion Date:\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""AUTHOR(S):\\n\"", \""COMPLETION DATE:\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\\n\"", \""Completion Date:\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -217,"""21a: Redact typoed Emails with indicator""","""\n when\n Section(searchText.contains(\""@\"") || searchText.toLowerCase().contains(\""mail\""))\n then\n section.redactByRegEx(\""mail[:\\\\.\\\\s]{1,2}([\\\\w\\\\/\\\\-\\\\{\\\\(\\\\. ]{3,20 }(@|a|f)\\\\s?[\\\\w\\\\/\\\\-\\\\{\\\\(\\\\. ]{3,20}(\\\\. \\\\w{2,4}\\\\b|\\\\.\\\\B|\\\\.\\\\w{1,4}\\\\b))\"",true,1,\""PII\"",21,\""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.2] -218,"""0: Combine address parts from ai to CBI_address (department is mandatory)""","""\n when\n Section(aiMatchesType(\""DEPARTMENT\""))\n then\n section.combineAiTypes(\""DEPARTMENT\"", \""ORG,STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\"", 20, \""CBI_address\"", 3, false);\n end""",[AI.1.0] -219,"""0: Combine address parts from ai to CBI_address (street is mandatory)""","""\n when\n Section(aiMatchesType(\""STREET\""))\n then\n section.combineAiTypes(\""STREET\"", \""ORG,DEPARTMENT,POSTAL,COUNTRY,CARDINAL,CITY,STATE\"", 20, \""CBI_address\"", 3, false);\n end""",[AI.1.0] -220,"""19b: Add recommendation for PII after Contact Person""","""\n when\n Section(searchText.toLowerCase().contains(\""contact person:\""))\n then\n section.recommendLineAfter(\""Contact Person\"", \""PII\"");\n section.recommendLineAfter(\""Contact person\"", \""PII\"");\n section.recommendLineAfter(\""contact person\"", \""PII\"");\n section.recommendLineAfter(\""Contact Person:\"", \""PII\"");\n section.recommendLineAfter(\""Contact person:\"", \""PII\"");\n section.recommendLineAfter(\""contact person:\"", \""PII\"");\n end""",[PII.13.0] -221,"""0: Combine address parts from ai to CBI_address (org is mandatory)""","""\n when\n Section(aiMatchesType(\""ORG\""))\n then\n section.combineAiTypes(\""ORG\"", \""DEPARTMENT,STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\"", 20, \""CBI_address\"", 3, false);\n end""",[AI.1.0] -222,"""25a: Redact phone numbers without indicators""","""\n when\n Section(text.contains(\""+\""))\n then\n section.redactByRegEx(\""(\\\\+[\\\\dO]{1,2} )(\\\\([\\\\dO]{1,3}\\\\))?[\\\\d\\\\-O ]{8,15}\"",false,0,\""PII\"",25,\""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.2] -223,"""34: Dossier""","""\n when\n Section(matchesType(\""dossier_redaction\""))\n then\n section.redact(\""dossier_redaction\"", 34, \""Dossier redaction found\"", \""Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)\"");\n end""","[ETC.4.0, ETC.4.1, ETC.4.2]" -224,"""30: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toUpperCase().contains(\""AUTHOR(S):\"")\n && searchText.toUpperCase().contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\"", \""Study Completion Date:\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\"", \""Study completion Date:\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""AUTHOR(S):\\n\"", \""STUDY COMPLETION DATE:\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\\n\"", \""Study Completion Date:\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\\n\"", \""Study completion Date:\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -225,"""30a: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toUpperCase().contains(\""AUTHOR(S)\"")\n && !searchText.toUpperCase().contains(\""AUTHOR(S):\"")\n && searchText.toUpperCase().contains(\""STUDY COMPLETION DATE\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S)\"", \""STUDY COMPLETION DATE\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\"", \""Study Completion Date\"", \""CBI_author\"", 30, true, \""Author(s) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\"", \""Study completion Date\"", \""CBI_author\"", 30, true, \""Author(s) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""AUTHOR(S)\\n\"", \""STUDY COMPLETION DATE\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\\n\"", \""Study Completion Date\"", \""CBI_author\"", 30, true, \""Author(s) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\\n\"", \""Study completion Date\"", \""CBI_author\"", 30, true, \""Author(s) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -226,"""29a: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toUpperCase().contains(\""AUTHOR(S)\"")\n && !searchText.toUpperCase().contains(\""AUTHOR(S):\"")\n && searchText.toUpperCase().contains(\""STUDY COMPLETION DATE\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S)\"", \""STUDY COMPLETION DATE\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\"", \""Study Completion Date\"", \""CBI_author\"", 29, true, \""Author(s) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\"", \""Study completion Date\"", \""CBI_author\"", 29, true, \""Author(s) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""AUTHOR(S)\\n\"", \""STUDY COMPLETION DATE\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\\n\"", \""Study Completion Date\"", \""CBI_author\"", 29, true, \""Author(s) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s)\\n\"", \""Study completion Date\"", \""CBI_author\"", 29, true, \""Author(s) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -227,"""28: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toUpperCase().contains(\""AUTHOR(S):\"")\n && searchText.toUpperCase().contains(\""COMPLETION DATE:\"")\n && !searchText.toUpperCase().contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""CBI_author\"", 28, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\"", \""Completion Date:\"", \""CBI_author\"", 28, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""AUTHOR(S):\\n\"", \""COMPLETION DATE:\"", \""CBI_author\"", 28, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n section.redactLinesBetween(\""Author(s):\\n\"", \""Completion Date:\"", \""CBI_author\"", 28, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -228,"""30b: Redact short Authors section""","""\n when\n Section(\n fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.toLowerCase().contains(\""author(s)\"")\n && searchText.length() < 50\n && sectionNumber <= 20\n && !aiMatchesType(\""CBI_Author\"")\n )\n then\n section.redactByRegEx(\""(?<=author\\\\(?s\\\\)?\\\\s\\\\n?)([\\\\p{Lu}\\\\p{L} ]{5,15}(,|\\\\n)?){1,3}\"",true,0,\""CBI_author\"",30,\""AUTHOR(S) was found\"",\""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[CBI.21.1] -229,"""26: Redact Phone and Fax by RegEx (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && (\n text.contains(\""Contact\"")\n || text.contains(\""Telephone\"")\n || text.contains(\""Téléphone\"")\n || text.contains(\""Phone\"")\n || text.contains(\""Fax\"")\n || text.contains(\""Tel\"")\n || text.contains(\""Ter\"")\n || text.contains(\""Mobile\"")\n || text.contains(\""Fel\"")\n || text.contains(\""Fer\"")\n ))\n then\n section.redactByRegEx(\""\\\\b(contact|t\\\\p{Ll}l\\\\p{Ll}phone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(O][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.O]{4,100}\\\\d)\\\\b\"", true, 2, \""PII\"", 26, \""Personal information found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.1] -230,"""60: Redact Must Impurities""","""\n when\n Section((fileAttributeByLabelEqualsIgnoreCase(\""Redact Impurities\"",\""Yes\"") || fileAttributeByLabelEqualsIgnoreCase(\""Redact Impurities\"",\""Y\"")) && matchesType(\""impurities\""))\n then\n section.redact(\""impurities\"", 60, \""Impurity found\"", \""specification_impurity\"");\n end""",[ETC.9.1] -231,"""21: Redact Emails by RegEx (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && searchText.contains(\""@\""))\n then\n section.redactByRegEx(\""\\\\b([A-Za-z0-9._%+\\\\-]+@\\\\s?[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\"", true, 1, \""PII\"", 21, \""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.2.0] -232,"""21a: Redact typoed Emails with indicator""","""\n when\n Section(searchText.contains(\""@\"") || searchText.toLowerCase().contains(\""mail\""))\n then\n section.redactByRegEx(\""mail[:\\\\.\\\\s]{1,2}([\\\\w\\\\/\\\\-\\\\{\\\\(\\\\. ]{3,20}(@|a|f)\\\\s?[\\\\w\\\\/\\\\-\\\\{\\\\(\\\\. ]{3,20}(\\\\. \\\\w{2,4}\\\\b|\\\\.\\\\B|\\\\.\\\\w{1,4}\\\\b))\"",true,1,\""PII\"",21,\""Personal information found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.1.2] -233,"""29: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""CBI_author\"", 29, true, \""AUTHOR(S) was found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -234,"""28: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""COMPLETION DATE:\"")\n && !searchText.contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""CBI_author\"", 28, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -235,"""30: Redact AUTHOR(S) (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""STUDY COMPLETION DATE:\"", \""CBI_author\"", 30, true, \""AUTHOR(S) was found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.1] -236,"""27: Redact AUTHOR(S) (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"")\n && searchText.contains(\""AUTHOR(S):\"")\n && searchText.contains(\""COMPLETION DATE:\"")\n && !searchText.contains(\""STUDY COMPLETION DATE:\"")\n )\n then\n section.redactLinesBetween(\""AUTHOR(S):\"", \""COMPLETION DATE:\"", \""CBI_author\"", 27, true, \""Author found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[PII.9.0] -237,"""34a: Redact dossier_redaction (Non vertebrate study)""","""\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""dossier_redaction\""))\n then\n section.redact(\""dossier_redaction\"", 34, \""Dossier dictionary entry found\"", \""Article 39(e)(3) of Regulation (EC) No 178/2002\"");\n end""",[ETC.12.0] -238,"""34b: Redact dossier_redaction (Vertebrate study)""","""\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\""Vertebrate Study\"",\""Yes\"") && matchesType(\""dossier_redaction\""))\n then\n section.redact(\""dossier_redaction\"", 34, \""Dossier dictionary entry found\"", \""Article 39(e)(2) of Regulation (EC) No 178/2002\"");\n end""",[ETC.12.1] diff --git a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java index 2fc8aa7c..c3629d9c 100644 --- a/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java +++ b/redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java @@ -25,9 +25,9 @@ public class RuleFileMigrationTest { // Put your redaction service drools paths and dossier-templates paths both RM and DM here static final List ruleFileDirs = List.of( - "/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools", - "/home/kschuettler/iqser/redaction/dossier-templates-v2", - "/home/kschuettler/iqser/fforesight/dossier-templates-v2"); + //"/Users/maverickstuder/Documents/DocuMine/dossier-templates-v2/", + "/Users/maverickstuder/Documents/RedactManager/dossier-templates-v2/"); + @Test diff --git a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt deleted file mode 100644 index de99d205..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/EFSA_sanitisation_GFL_v1/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;\nimport com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;\nimport com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualRedactionApplicationService manualRedactionApplicationService\nglobal NerEntitiesAdapter nerEntitiesAdapter\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//------------------------------------ Syngenta specific rules ------------------------------------\n\n// Rule unit: SYN.1\nrule \"SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n $section: Section(containsString(\"CT\") || containsString(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n entityCreationService.byRegexIgnoreCase(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.skip(\"SYN.1.0\", \"\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n\n\n//------------------------------------ CBI rules ------------------------------------\n\n// Rule unit: CBI.0\nrule \"CBI.0.0: Redact CBI Authors (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.0\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"CBI.0.1: Redact CBI Authors (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_author\", dictionaryEntry)\n then\n $entity.apply(\"CBI.0.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.1\nrule \"CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.skip(\"CBI.1.0\", \"Address found for Non Vertebrate Study\");\n end\n\nrule \"CBI.1.1: Redact CBI Address (Vertebrate Study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $entity: RedactionEntity(type == \"CBI_address\", dictionaryEntry)\n then\n $entity.apply(\"CBI.1.1\", \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: CBI.2\nrule \"CBI.2.0: Don't redact genitive CBI_author\"\n when\n $entity: RedactionEntity(type == \"CBI_author\", anyMatch(textAfter, \"['’’'ʼˈ´`‘′ʻ’']s\"), isApplied())\n then\n RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), \"CBI_author\", EntityType.FALSE_POSITIVE, document);\n falsePositive.skip(\"CBI.2.0\", \"Genitive Author found\");\n insert(falsePositive);\n end\n\n\n// Rule unit: CBI.7\nrule \"CBI.7.0: Do not redact Names and Addresses if published information found in section without tables\"\n when\n $section: Section(!hasTables(),\n hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $section.getEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.0\",\n \"Published Information found in section\",\n $section.getEntitiesOfType(\"published_information\")\n );\n });\n end\n\nrule \"CBI.7.1: Do not redact Names and Addresses if published information found in same table row\"\n when\n $table: Table(hasEntitiesOfType(\"published_information\"),\n (hasEntitiesOfType(\"CBI_author\") || hasEntitiesOfType(\"CBI_address\")))\n then\n $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(\"CBI_author\", \"CBI_address\"))\n .forEach(redactionEntity -> {\n redactionEntity.skipWithReferences(\n \"CBI.7.1\",\n \"Published Information found in row\",\n $table.getEntitiesOfTypeInSameRow(\"published_information\", redactionEntity)\n );\n });\n end\n\n\n// Rule unit: CBI.9\nrule \"CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.0\", \"Author(s) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.9.1\", \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.10\nrule \"CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author(s)\"))\n then\n $table.streamTableCellsWithHeader(\"Author(s)\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.0\", \"Author(s) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\nrule \"CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $table: Table(hasHeader(\"Author\"))\n then\n $table.streamTableCellsWithHeader(\"Author\")\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"CBI_author\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(redactionEntity -> {\n redactionEntity.apply(\"CBI.10.1\", \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n redactionEntity.addEngine(Engine.RULE);\n insert(redactionEntity);\n });\n end\n\n\n// Rule unit: CBI.11\nrule \"CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -1\n when\n $table: Table(hasHeader(\"Author(s)\") && hasHeader(\"Vertebrate Study Y/N\"))\n then\n $table.getEntitiesOfType(\"CBI_author\").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));\n end\n\n\n// Rule unit: CBI.16\nrule \"CBI.16.0: Add CBI_author with \\\"et al.\\\" Regex (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.0\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n insert(entity);\n });\n end\n\nrule \"CBI.16.1: Add CBI_author with \\\"et al.\\\" Regex (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"et al.\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", \"CBI_author\", EntityType.ENTITY, 1, $section)\n .forEach(entity -> {\n entity.apply(\"CBI.16.1\", \"Author found by \\\"et al\\\" regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n entity.addEngine(Engine.RULE);\n insert(entity);\n dictionary.addLocalDictionaryEntry(\"CBI_author\", entity.getValue(), false);\n });\n end\n\n\n// Rule unit: CBI.17\nrule \"CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species\") && containsString(\"Source\") && !containsString(\"Species:\") && !containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.0\", \"Line after \\\"Source\\\" in Test Organism Section\");\n insert(entity);\n });\n end\n\nrule \"CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon\"\n when\n $section: Section(!hasTables(), containsString(\"Species:\"), containsString(\"Source:\"))\n then\n entityCreationService.lineAfterString(\"Source:\", \"CBI_address\", EntityType.RECOMMENDATION, $section)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n entity.skip(\"CBI.17.1\", \"Line after \\\"Source:\\\" in Test Animals Section\");\n insert(entity);\n });\n end\n\n\n// Rule unit: CBI.20\nrule \"CBI.20.0: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (non vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.skip(\"CBI.20.0\", \"PERFORMING LABORATORY was found for non vertebrate study\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\nrule \"CBI.20.1: Redact between \\\"PERFORMING LABORATORY\\\" and \\\"LABORATORY PROJECT ID:\\\" (vertebrate study)\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $section: Section(!hasTables(), containsString(\"PERFORMING LABORATORY:\"), containsString(\"LABORATORY PROJECT ID:\"))\n then\n entityCreationService.betweenStrings(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", EntityType.ENTITY, $section)\n .forEach(laboratoryEntity -> {\n laboratoryEntity.apply(\"CBI.20.1\", \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n laboratoryEntity.addEngine(Engine.RULE);\n dictionary.addLocalDictionaryEntry(laboratoryEntity);\n insert(laboratoryEntity);\n });\n end\n\n\n//------------------------------------ PII rules ------------------------------------\n\n// Rule unit: PII.0\nrule \"PII.0.0: Redact all PII (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.0\", \"Personal Information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"PII.0.1: Redact all PII (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $pii: RedactionEntity(type == \"PII\", dictionaryEntry)\n then\n $pii.apply(\"PII.0.1\", \"Personal Information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: PII.1\nrule \"PII.1.0: Redact Emails by RegEx (Non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.0\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\nrule \"PII.1.1: Redact Emails by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"@\"))\n then\n entityCreationService.byRegex(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", \"PII\", EntityType.ENTITY, 1, $section)\n .forEach(emailEntity -> {\n emailEntity.addEngine(Engine.RULE);\n emailEntity.apply(\"PII.1.1\", \"Found by Email Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(emailEntity);\n });\n end\n\n\n// Rule unit: PII.2\nrule \"PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.0\", \"Found by Phone and Fax Regex\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\nrule \"PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(containsString(\"Contact\") ||\n containsString(\"Telephone\") ||\n containsString(\"Phone\") ||\n containsString(\"Ph.\") ||\n containsString(\"Fax\") ||\n containsString(\"Tel\") ||\n containsString(\"Ter\") ||\n containsString(\"Mobile\") ||\n containsString(\"Fel\") ||\n containsString(\"Fer\"))\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\b(contact|telephone|phone|ph\\\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", \"PII\", EntityType.ENTITY, 2, $section)\n .forEach(contactEntity -> {\n contactEntity.addEngine(Engine.RULE);\n contactEntity.apply(\"PII.2.1\", \"Found by Phone and Fax Regex\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n insert(contactEntity);\n });\n end\n\n\n// Rule unit: PII.9\nrule \"PII.9.0: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.0\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.1: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"COMPLETION DATE:\"), !containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.1\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.2: Redact between \\\"AUTHOR(S)\\\" and \\\"COMPLETION DATE\\\" (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.2\", \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\nrule \"PII.9.3: Redact between \\\"AUTHOR(S)\\\" and \\\"STUDY COMPLETION DATE\\\" (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value.toLowerCase() == \"yes\")\n $section: Section(!hasTables(), containsString(\"AUTHOR(S):\"), containsString(\"STUDY COMPLETION DATE:\"))\n then\n entityCreationService.betweenStrings(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", EntityType.ENTITY, $section)\n .forEach(authorEntity -> {\n authorEntity.apply(\"PII.9.3\", \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n authorEntity.addEngine(Engine.RULE);\n insert(authorEntity);\n });\n end\n\n\n//------------------------------------ Other rules ------------------------------------\n\n// Rule unit: ETC.0\nrule \"ETC.0.0: Purity Hint\"\n when\n $section: Section(containsStringIgnoreCase(\"purity\"))\n then\n entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", "hint_only", EntityType.HINT, 1, $section)\n .forEach(hint -> {\n hint.addEngine(Engine.RULE);\n hint.skip(\"ETC.0.0\", \"\");\n });\n end\n\n\n// Rule unit: ETC.2\nrule \"ETC.2.0: Redact signatures (non vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.2.0: Redact signatures (vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $signature: Image(imageType == ImageType.SIGNATURE)\n then\n $signature.apply(\"ETC.2.0\", \"Signature Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.3\nrule \"ETC.3.0: Redact logos (vertebrate study)\"\n when\n not FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.0\", \"Logo Found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"ETC.3.1: Redact logos (non vertebrate study)\"\n when\n FileAttribute(label == \"Vertebrate Study\", value == \"Yes\")\n $logo: Image(imageType == ImageType.LOGO)\n then\n $logo.apply(\"ETC.3.1\", \"Logo Found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// Rule unit: ETC.5\nrule \"ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n not FileAttribute(label == \"Confidentiality\", value == \"confidential\")\n $dossierRedaction: RedactionEntity(type == \"dossier_redaction\")\n then\n $dossierRedaction.removeFromGraph();\n retract($dossierRedaction);\n end\n\n\n//------------------------------------ AI rules ------------------------------------\n\n// Rule unit: AI.0\nrule \"AI.0.0: add all NER Entities of type CBI_author\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"CBI_author\"))\n then\n nerEntities.streamEntitiesOfType(\"CBI_author\")\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))\n .forEach(entity -> insert(entity));\n end\n\n\n// Rule unit: AI.1\nrule \"AI.1.0: combine and add NER Entities as CBI_address\"\n salience 999\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"ORG\") || hasEntitiesOfType(\"STREET\") || hasEntitiesOfType(\"CITY\"))\n then\n nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)\n .map(boundary -> entityCreationService.byBoundary(boundary, \"CBI_address\", EntityType.RECOMMENDATION, document))\n .forEach(entity -> {\n entity.addEngine(Engine.NER);\n insert(entity);\n });\n end\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId)\n $entityToBeResized: RedactionEntity(matchesAnnotationId($id))\n then\n manualRedactionApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.setIgnored(true);\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)\n not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.setIgnored(true);\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)\n $entityToForce: RedactionEntity(matchesAnnotationId($id))\n then\n $entityToForce.apply(\"MAN.2.0\", \"Forced redaction\", $legalBasis);\n $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply image recategorization\"\n salience 128\n when\n ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)\n $image: Image($id == id)\n then\n $image.setImageType(ImageType.fromString($imageType));\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: RedactionEntity($type: type, $entityType: entityType)\n $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $contained.removeFromGraph();\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)\n $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $first.removeFromGraph();\n $second.removeFromGraph();\n RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);\n retract($first);\n retract($second);\n insert(mergedEntity);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)\n $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.removeFromGraph();\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)\n $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: RedactionEntity(entityType == EntityType.ENTITY)\n $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $recommendation.removeFromGraph();\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)\n $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)\n then\n $lowerRank.removeFromGraph();\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n entity.addEngine(Engine.RULE);\n insert(entity);\n });\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/addNewRulesHere.drl b/redaction-service-v1/rules-management/src/test/resources/addNewRulesHere.drl deleted file mode 100644 index e69de29b..00000000 diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt deleted file mode 100644 index 2020709c..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dev/Basf-Demo/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n\n//---------------------------------------------------------------------------\n\n\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.0.1: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n\n then\n $tocHeadline.getParent().getPages()\n .forEach(page -> page.getMainBody().stream()\n .filter(node -> !node.getType().equals(NodeType.IMAGE))\n .filter(node -> node.getPages().stream().noneMatch(nodePage -> nodePage.getNumber() < page.getNumber()))\n .forEach(node -> retract(node))\n );\n end\n \n\n\n/*\nrule \"H.0.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.RECOMMENDATION);\n end\n*/\n\nrule \"DOC.0.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"408\",\"414\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n onPage(1)\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline, Method No. (\\\\d{3})\", 1, $section.getTextBlock()) \n ).flatMap(Collection::stream).findFirst()\n .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n onPage(1)\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// Examples found in PoC 1\n// entityCreationService.byRegex(\"((OECD Guidelines for Testing of Chemicals, Procedure)|(OECD Guidelines for the Testing of Chemicals No\\\\.)|(OECD Test Guideline)|(OECD \\\\[Test Guideline, Number)) \\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n// entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n// );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\")\n );\n// new approach OECD Guideline\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\")\n );\n// missing OECD guideline rules for RFP demo file\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.1: Guidelines\"\n when\n $headline: Headline(\n onPage(1),\n containsString(\"OECD\")\n )\n then\n entityCreationService.byRegex(\"(OECD (No\\\\.? )?(\\\\d{3})( \\\\(\\\\d{4}\\\\))?)\", \"oecd_guideline\", EntityType.ENTITY,1, $headline).forEach(entity ->\n entity.apply(\"DOC.1.1\", \"OECD Guideline found\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\");\n });\n end\n\n\nrule \"DOC.1.3: Remove guidelines from irrelevant sections\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.remove(\"DOC.1.3\", \"remove guidelines from irrelevant sections\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.3.2: Experimental Completion Date\"\n salience 10\n when \n $section: Section(onPage(1) && (containsString(\"STUDY COMPLETED ON\") || containsString(\"STUDY COMPLETION DATE\") || containsString(\"Report completion date\") || containsString(\"Date of Report\") || containsString(\"AMENDMENT COMPLETION DATE\") || containsString(\"AMENDMENT COMPLETED ON\")))\n \n then\n entityCreationService.byRegex(\"STUDY COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"STUDY COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Report completion date (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"Date of Report (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETION DATE (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n entityCreationService.byRegex(\"AMENDMENT COMPLETED ON (.{5,20}\\\\d{4})\", \"experimental_end_date\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.3.2\", \"Experimental end date found\");\n });\n end\n\n\n\n\n\n // hide all skipped species and strains except in the relevant sections\n rule \"DOC.4.2: Species\"\n salience 1\n when\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.remove(\"DOC.4.2\",\"n-a\");\n retract(entity);\n });\n end\n\n\nrule \"DOC.4.3: Species\"\n when\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.4.3\", \"Species found.\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.0: Strain\"\n when\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.0\", \"Strain found.\");\n });\n end\n\n\n\n\nrule \"DOC.35.0: Sex\"\n when\n \n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"Species and strain\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.35.0\", \"Test animal sex found\");\n });\n end\n\n\n\nrule \"DOC.6.0: Authors\"\n when\n $headline: Headline(onPage(1), containsString(\"AUTHOR\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"author\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.6.0\", \"Author found\"));\n end\n\n\n\nrule \"DOC.6.2: Authors\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"AUTHOR(S)\" || contains \"AUTHORS\" || contains \"Author\"), getMainBodyTextBlock().getSearchText() (contains \"STUDY COMPLETED ON\" || contains \"STUDY COMPLETION DATE\" || contains \"DATE OF INTERIM REPORT\" || contains \"Report completion date\" || contains \"Date of Report\" || contains \"AMENDMENT COMPLETION DATE\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"AUTHOR(S)\", \"AUTHORS\", \"Author\"),\n List.of(\"STUDY COMPLETED ON\", \"STUDY COMPLETION DATE\", \"DATE OF INTERIM REPORT\", \"Report completion date\", \"Date of Report\", \"AMENDMENT COMPLETION DATE\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.2\", \"Author found\"));\n end\n\n\n\n\nrule \"DOC.6.6: laboratory_project_identification\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Laboratory Project Identification\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\"),\n List.of(\"SPONSOR\", \"VOLUME\", \"This\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.6.6\", \"Laboratory Project Identification\"));\n end\n\n\n\nrule \"DOC.7.2: study title by document structure\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"STUDY TITLE\" || contains \"Study Title\" || contains \"STUDYTITLE\" || contains \"Report (Final)\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"STUDY TITLE\", \"STUDYTITLE\", \"Report (Final)\"),\n List.of(\"TEST GUIDELINES\", \"TEST GUIDELINE(S)\", \"Guidelines\", \"DATA REQUIREMENT\", \"AUTHOR(S)\", \"AUTHOR\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.7.2\", \"Study title found\"));\n end\n\n\n\nrule \"DOC.8.1: Performing Laboratory\"\n when\n $page: Page(getNumber() == 1, getMainBodyTextBlock().getSearchText() (contains \"PERFORMING LABORATORY\" || contains \"TEST FACILITIES\" || contains \"TEST FACILITY\" || contains \"Test Facility\"), getMainBodyTextBlock().getSearchText() (contains \"LABORATORY PROJECT IDENTIFICATION\" || contains \"TEST FACILITY PROJECT IDENTIFICATION\" || contains \"Sponsor\"))\n then\n entityCreationService.shortestBetweenAnyString(\n List.of(\"PERFORMING LABORATORY\", \"TEST FACILITIES\", \"TEST FACILITY\"),\n List.of(\"LABORATORY PROJECT IDENTIFICATION\", \"TEST FACILITY PROJECT IDENTIFICATION\", \"Sponsor\", \"PROJECT IDENTIFICATION\"),\n \"author\",\n EntityType.ENTITY,\n document)\n .forEach(entity -> entity.apply(\"DOC.8.1\", \"Performing Laboratory found\"));\n end\n\n\n\n\nrule \"DOC.8.2: Summary Methods\"\n when\n $headline: Headline(containsString(\"1.1. METHODS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_methods\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.2\", \"Summary Methods found\"));\n end\n\nrule \"DOC.8.3: Summary Observations Laboratory\"\n when\n $headline: Headline(containsString(\"1.2. OBSERVATIONS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_observations\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.3\", \"Summary Observations found\"));\n end\n\n\nrule \"DOC.8.5: Summary Results\"\n when\n Headline((containsStringIgnoreCase(\"1.3. RESULTS\") || containsStringIgnoreCase(\"1.2. RESULTS\")), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> {\n entity.apply(\"DOC.8.5\", \"Summary Results\");\n });\n end\n\nrule \"DOC.8.6: Summary Results 2\"\n when\n $headline: Headline(containsString(\"1.2. RESULTS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"summary_results\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\"))\n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\")) \n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.6\", \"Summary Results\"));\n end\n\n\n\nrule \"DOC.8.4: Summary Conclusion\"\n when\n $headline: Headline(containsString(\"1.4. CONCLUSION\") || containsString(\"1.3. CONCLUSION\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_conclusion\", EntityType.ENTITY)\n .filter(e -> !e.getValue().contains(\"Report; Project No\")) \n .filter(e -> !e.getValue().startsWith(\"This document\")) \n .filter(e -> !e.getValue().startsWith(\"Page\"))\n .filter(e -> !e.getValue().startsWith(\"2. INTRODUCTION\"))\n .filter(e -> !e.getValue().startsWith(\"BASF\"))\n .filter(e -> !e.getValue().startsWith(\"The Chemical Company\"))\n .filter(e -> !e.getValue().startsWith(\"We create chemistry\")) \n .forEach(entity -> entity.apply(\"DOC.8.4\", \"Summary Conculsion found\"));\n end\n\n\n\nrule \"DOC.9.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\")\n || containsString(\"GLP CERTIFICATE (FROM THE COMPETENT AUTHORITY)\")\n || containsString(\"GLP COMPLIANCE STATEMENT\")\n || containsString(\"GLP STATEMENT\") \n )\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.9.0\", \"GLP Study found\");\n });\n end\n\n\n\nrule \"DOC.9.1: GLP Study\"\n when\n $paragraph: Paragraph(containsString(\"GLP COMPLIANCE STATEMENT\"))\n then\n entityCreationService.byRegex(\"GLP COMPLIANCE STATEMENT\", \"glp_study\", EntityType.ENTITY, $paragraph).forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"GLP Study found\");\n });\n end\n\n\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.getManualOverwrite().addChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);\n end\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.1\nrule \"X.1.0: merge intersecting Entities of same type\"\n salience 64\n when\n $first: TextEntity($type: type, $entityType: entityType, !resized(), active())\n $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())\n then\n TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);\n $first.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n $second.remove(\"X.1.0\", \"merge intersecting Entities of same type\");\n retract($first);\n retract($second);\n mergedEntity.getIntersectingNodes().forEach(node -> update(node));\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.6\nrule \"X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY\"\n salience 32\n when\n $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())\n then\n $lowerRank.getIntersectingNodes().forEach(node -> update(node));\n $lowerRank.remove(\"X.6.0\", \"remove Entity of lower rank, when intersected by entity of type ENTITY\");\n retract($lowerRank);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n//------------------------------------ Local dictionary search rules ------------------------------------\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt deleted file mode 100644 index 1739decc..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dev/Flora/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport static java.lang.String.format;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;\nimport static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;\n\nimport java.util.List;\nimport java.util.LinkedList;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.Collection;\nimport java.util.stream.Stream;\nimport java.util.Optional;\n\nimport com.iqser.red.service.redaction.v1.server.model.document.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.TextRange;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity\nimport com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.*;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;\nimport com.iqser.red.service.redaction.v1.server.model.NerEntities;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;\nimport com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;\nimport com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;\nimport com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;\n\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;\nimport com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;\n\nglobal Document document\nglobal EntityCreationService entityCreationService\nglobal ManualChangesApplicationService manualChangesApplicationService\nglobal Dictionary dictionary\n\n//------------------------------------ queries ------------------------------------\n\nquery \"getFileAttributes\"\n $fileAttribute: FileAttribute()\n end\n//---------------------------------------------------------------------------\n\nrule \"H.0.0: retract table of contents page\"\n when\n $page: Page(getMainBodyTextBlock().getSearchText().contains(\"........\") || (getMainBodyTextBlock().getSearchText().contains(\"APPENDICES\") && getMainBodyTextBlock().getSearchText().contains(\"TABLES\")))\n $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)\n then\n retract($node);\n end\n\n\nrule \"H.1.0: Ignore Table of Contents\"\n salience 10\n when\n $tocHeadline: Headline(containsString(\"CONTENTS\"))\n $page: Page() from $tocHeadline.getParent().getPages()\n $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))\n then\n retract($node);\n end\n\n\n// Rule unit: MAN.0\nrule \"H.2.0: Show headlines\"\n when\n $headline: Headline()\n then\n entityCreationService.bySemanticNode($headline, \"headline\", EntityType.ENTITY);\n end\n\n\nrule \"H.3.0: Study Type File Attribute\"\n when\n not FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"438\",\"439\",\"471\",\"487\"))\n $section: Section(\n (containsString(\"DATA REQUIREMENT\") || containsString(\"TEST GUIDELINE\") || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\"))\n ,(containsString(\"OECD\") || containsString(\"EPA\") || containsString(\"OPPTS\"))\n )\n then\n Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\\\d]{2})\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", 1, $section.getTextBlock()),\n RedactionSearchUtility.findTextRangesByRegexIgnoreCase(\"(?<=OECD) Guideline (4\\\\d{2})\", 1, $section.getTextBlock())).flatMap(Collection::stream).findFirst()\n .map(textRange -> $section.getTextBlock().subSequence(textRange).toString())\n .map(value -> FileAttribute.builder().label(\"OECD Number\").value(value).build())\n .ifPresent(fileAttribute -> insert(fileAttribute));\n end\n\n\nrule \"DOC.1.0: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n then\n entityCreationService.byRegex(\"OECD (No\\\\.? )?\\\\d{3}( \\\\(\\\\d{4}\\\\))?\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD[\\\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\\\d{3,4}(?>.{1,100}\\\\d{4}\\\\))?|\\\\[.{1,20}.Skin.{1,20}\\\\]|[\\\\d\\\\s,\\\\(\\\\)]{7,10}|[\\\\w\\\\.\\\\s]{1,15}[\\\\d]{3}\\\\s\\\\(\\\\d{4}\\\\)|.{0,20}[N|n]umber\\\\s\\\\d{3}.{0,1}|Test Guideline \\\\d{3})\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EPA (OPPTS )?\\\\d{3}[. ]\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"EC (Directive )?(No\\\\.? )?\\\\d{3,4}\\\\/\\\\d{3,4}((,? B(\\\\.| )\\\\d{1,2}\\\\.?)? \\\\(\\\\d{4}\\\\))?\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"Commission Regulation \\\\(EC\\\\) No \\\\d{3}\\\\/\\\\d{4}\", \"ec_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EC Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Method 4\\\\d{2}.{5,40}\\\\(.{5,40}\\\\d{4}\\\\)\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OPPTS (Guideline Number )?\\\\d{3}\\\\.\\\\d{4}( \\\\(\\\\d{4}\\\\))?\", \"epa_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"EPA Guideline found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline no. found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)(?:[\\\\w\\\\s,\\\\[\\\\]\\\\(\\\\)\\\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\\\d]{2}),?\\\\s\\\\(?(\\\\d{4})\\\\)?\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD)[\\\\w\\\\s,\\\\[\\\\]]{1,10}\\\\((\\\\d{4})\\\\)\\\\s(4[\\\\d]{2})\", \"oecd_guideline_year\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD).{5,40}Method (4[\\\\d]{2}).{1,65}(\\\\d{4})\\\\)\", \"oecd_guideline_year\", EntityType.ENTITY,2, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline year found\", \"n-a\")\n );\n entityCreationService.byRegex(\"(?<=OECD) Guideline (4\\\\d{2})\", \"oecd_guideline_number\", EntityType.ENTITY,1, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline number found\", \"n-a\")\n );\n entityCreationService.byRegex(\"OECD Guideline 4\\\\d{2}\", \"oecd_guideline\", EntityType.ENTITY, $section).forEach(entity ->\n entity.apply(\"DOC.1.0\", \"OECD Guideline found\", \"n-a\")\n );\n end\n\n\nrule \"DOC.1.2: Guidelines\"\n when\n $section: Section(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n && (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\",\"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.apply(\"DOC.1.2\", \"OECD guideline found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.1.3: Guidelines\"\n when\n $section: Section(\n (\n hasEntitiesOfType(\"oecd_guideline\")\n || hasEntitiesOfType(\"epa_guideline\")\n || hasEntitiesOfType(\"ec_guideline\")\n )\n && !(\n (\n containsString(\"DATA REQUIREMENT\")\n || containsString(\"TEST GUIDELINE\")\n || containsString(\"MÉTODO(S) DE REFERÊNCIA(S):\")\n )\n && (\n containsString(\"OECD\")\n || containsString(\"EPA\")\n || containsString(\"OPPTS\")\n )\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"oecd_guideline\", \"ec_guideline\", \"epa_guideline\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.2.0: Report number\"\n when\n $section: Section(containsString(\"LABORATORY PROJECT ID\") , containsString(\"Report Number:\"))\n then\n entityCreationService.lineAfterString(\"Report Number:\", \"report_number\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.2.0\", \"Report number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.3.0: Experimental Starting Date\"\n when\n $section: Section(containsString(\"Experimental I. Starting Date:\") || containsString(\"Experimental II. Starting Date:\") || containsStringIgnoreCase(\"experimental start date\") || containsStringIgnoreCase(\"experimental starting date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental start date\",\n \"Experimental start date:\",\n \"Experimental Starting Date\",\n \"Experimental Starting Date:\",\n \"Experimental starting date\",\n \"Experimental starting date:\",\n \"Experimental Start Date\",\n \"Experimental Start Date:\",\n \"Experimental I. Starting Date:\",\n \"Experimental II. Starting Date:\"), \"experimental_start_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.3.0\", \"Experimental start date found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.4.0: Experimental Completion Date\"\n when\n $section: Section(containsStringIgnoreCase(\"experimental termination date\") || containsStringIgnoreCase(\"experimental completion date\"))\n then\n entityCreationService.lineAfterStrings(\n List.of(\"Experimental termination date\",\n \"Experimental termination date:\",\n \"Experimental Completion Date\",\n \"Experimental Completion Date:\",\n \"Experimental completion date\",\n \"Experimental completion date:\",\n \"Experimental Termination Date\",\n \"Experimental Termination Date:\"), \"experimental_end_date\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.4.0\", \"Experimental end date found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.5.0: Ignore species and strain in irrelevant study types\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"471\",\"474\",\"487\"))\n $section: Section(hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\n rule \"DOC.5.1: Hide all skipped species and strains except in the relevant sections\"\n salience 1\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n (hasEntitiesOfType(\"species\") || hasEntitiesOfType(\"strain\"))\n && !(\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(List.of(\"species\", \"strain\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.5.2: Species\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(hasEntitiesOfType(\"species\"))\n then\n $section.getEntitiesOfType(\"species\").forEach(entity -> {\n entity.apply(\"DOC.5.2\", \"Species found.\", \"n-a\");\n entity.setValue(entity.getValue().toLowerCase());\n });\n end\n\n\nrule \"DOC.5.3: Strain\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\"))\n $section: Section(\n hasEntitiesOfType(\"species\")\n && hasEntitiesOfType(\"strain\")\n && (\n anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"animals\")\n || anyHeadlineContainsStringIgnoreCase(\"specification\")\n )\n )\n then\n $section.getEntitiesOfType(\"strain\").forEach(entity -> {\n entity.apply(\"DOC.5.3\", \"Strain found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.0: study title by document structure\"\n when\n $table: Table(onPage(1),\n (containsString(\"Final Report\") || containsString(\"SPL\")),\n numberOfRows == 1,\n numberOfCols == 1)\n then\n\n entityCreationService.bySemanticNode($table.getCell(0, 0).streamChildren().toList().get(1), \"title\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.6.0\", \"Study title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.1: study title\"\n when\n $table: Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $table).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.1\", \"Title found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.6.2: study title\"\n when\n not Table(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n $section: Section(onPage(1), (containsString(\"Final Report\") || containsString(\"SPL\")))\n then\n entityCreationService.byRegexWithLineBreaksIgnoreCase(\"(?<=\\\\n)[\\\\w\\\\W]{1,300}(?=\\\\nFinal Report)\", \"title\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.6.2\", \"Title found\", \"n-a\");\n });\n end\n\n\n\nrule \"DOC.7.0: Performing Laboratory (Name)\"\n when\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n entityCreationService.lineAfterString(\"PERFORMING LABORATORY:\", \"laboratory_name\", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {\n entity.apply(\"DOC.7.0\", \"Performing Laboratory found\", \"n-a\");\n });\n end\n\n\n rule \"DOC.7.1: Performing Laboratory (Country)\"\n when\n nerEntities: NerEntities(hasEntitiesOfType(\"COUNTRY\"))\n $section: Section(containsString(\"PERFORMING LABORATORY:\"))\n then\n nerEntities.streamEntitiesOfType(\"COUNTRY\")\n .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))\n .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, \"laboratory_country\", EntityType.ENTITY, $section))\n .forEach(entity -> {\n entity.apply(\"DOC.7.1\", \"Performing Laboratory found\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.7.2: Performing Laboratory (Country & Name) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && (containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(\"laboratory_country\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory country dictionary entry found.\", \"n-a\");\n });\n $section.getEntitiesOfType(\"laboratory_name\").forEach(entity -> {\n entity.apply(\"DOC.7.2\", \"Performing laboratory name dictionary entry found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.7.3: Performing Laboratory (Country) from dict\"\n when\n $section: Section(\n (hasEntitiesOfType(\"laboratory_country\") || hasEntitiesOfType(\"laboratory_name\"))\n && !(containsString(\"PERFORMING LABORATORY:\") || (containsString(\"PERFORMING\") && containsString(\"LABORATORY:\")))\n )\n then\n $section.getEntitiesOfType(List.of(\"laboratory_country\", \"laboratory_name\")).forEach(entity -> {\n entity.removeFromGraph();\n retract(entity);\n });\n end\n\n\nrule \"DOC.8.0: GLP Study\"\n when\n $headline: Headline(containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE\")\n || containsString(\"GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT\")\n || (containsString(\"DECLARACAO DE CONFORMIDADE\") && containsString(\"PRATICAS DE LABORATORIO\"))\n || containsString(\"GLP Certificate\")\n || containsString(\"GLP Certificates\")\n || containsString(\"GOOD LABORATORY PRACTICE (GLP) CERTIFICATE\")\n || containsString(\"Good Laboratory Practice Certificate\")\n || containsString(\"STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION\"))\n then\n entityCreationService.bySemanticNode($headline, \"glp_study\", EntityType.ENTITY).ifPresent(entity -> {\n entity.apply(\"DOC.8.0\", \"GLP Study found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.0: Batch number from CoA\"\n when\n $section: Section(\n (\n anyHeadlineContainsString(\"Analytical Report\")\n || anyHeadlineContainsStringIgnoreCase(\"Certificate of Analysis\")\n || containsStringIgnoreCase(\"Certificate of Analysis\")\n )\n && (\n containsStringIgnoreCase(\"batch\")\n || containsStringIgnoreCase(\"bath\")\n || containsStringIgnoreCase(\"barch\")\n || containsStringIgnoreCase(\"bateb\")\n )\n && (\n containsStringIgnoreCase(\"identification\")\n || containsStringIgnoreCase(\"ldentitfication\")\n || containsStringIgnoreCase(\"wentification\")\n || containsStringIgnoreCase(\"mentification\")\n || containsStringIgnoreCase(\"kientification\")\n || containsStringIgnoreCase(\"reference number\")\n || containsStringIgnoreCase(\"test substance\")\n )\n )\n then\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"(Batch Identification):\",\n \"Bateb Identification\",\n \"Batch Wentification\",\n \"Batch Mentification\",\n \"Batch Kientification\",\n \"Barch Identification\",\n \"Bath ldentitfication\",\n \"Batch of test substance :\"), \"batch_number\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.9.0\", \"Batch number found in CoA\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.1: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n then\n Stream.of(entityCreationService.byRegex(\"Batch ID ([A-Z\\\\d\\\\-]{7,14})\", \"batch_number\", EntityType.ENTITY, 1, $section),\n entityCreationService.lineAfterStrings(List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\"\n ), \"batch_number\", EntityType.ENTITY, $section)).flatMap(a -> a)\n .forEach(entity -> {\n entity.apply(\"DOC.9.1\", \"Batch number found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.9.2: Batch number\"\n when\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Test Substance\")\n || anyHeadlineContainsStringIgnoreCase(\"Test and Control Substances\")\n || anyHeadlineContainsStringIgnoreCase(\"Test Item\")\n )\n && !(\n anyHeadlineContainsString(\"component\")\n || anyHeadlineContainsString(\"reference\")\n || anyHeadlineContainsString(\"blank\")\n )\n && containsStringIgnoreCase(\"batch\")\n )\n $batchNumber: String() from List.of(\"Batch Identification\",\n \"Batch number:\",\n \"Batch reference number:\",\n \"Batch:\",\n \"Batch/Lot number:\",\n \"Batch (Lot) Number:\",\n \"Batch Number:\",\n \"Batch Nº:\",\n \"Batch no:\")\n $table: Table(containsStringIgnoreCase($batchNumber)) from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()\n then\n entityCreationService.lineAfterStringAcrossColumnsIgnoreCase($batchNumber, \"batch_number\", EntityType.ENTITY, $table).forEach(entity -> {\n entity.apply(\"DOC.9.2\", \"Batch number found\", \"n-a\");\n });\n end\n\n\n\n\nrule \"DOC.10.0: Conclusions - LD50, LC50, Confidence\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"425\",\"436\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"Conclusion\") || anyHeadlineContainsStringIgnoreCase(\"Lethality\"))\n && (containsString(\"LD\") || containsString(\"LC\") || containsString(\"50\") || containsString(\"LD50\") || containsString(\"lethal concentration\") || containsString(\"lethal dose\"))\n && (\n containsString(\"greater than\")\n || containsString(\"higher than\")\n || containsString(\"above\")\n || containsString(\"in excess\")\n || containsString(\"exceeds\")\n || containsString(\"was found to be\")\n || containsString(\"was calculated to be\")\n || containsString(\"estimated to be\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(L[D|C]\\\\s?50|lethal concentration|lethal dose).{1,200}(greater than|considered to be above|in excess of|exceeds|higher than)\", \"ld50_greater\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 greater than found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"\\\\b(?:(?:greater|higher) than|considered to be above|(?:was|is) (?:found|estimated) to be|was calculated to be|in excess of|exceeds|equal to)\\\\s?([\\\\d\\\\.]{1,6})\\\\s?mg\\\\/(?:kg|L)\", \"ld50_value\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"LD50 value found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_minimal\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Minimal Confidence found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"confidence interval (?:is )?([\\\\d\\\\.]{2,6}).{0,20} to (?:greater than )?([\\\\d\\\\.]{2,6})\", \"confidence_maximal\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.10.0\", \"Maximal Confidence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.0: Guideline Deviation\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n (getHeadline().containsStringIgnoreCase(\"General Information\") || containsString(\"GENERAL INFORMATION\"))\n && (containsStringIgnoreCase(\"from the\") || containsStringIgnoreCase(\"to the\"))\n )\n then\n entityCreationService.betweenRegexes(\"(?:Deviations? from the [G|g]uidelines?)(?: and| or)?( the)?(?: Study Plan)?\", \"(?:(?:Deviations? from the Study Plan)|(?:Performing laboratory test)|(?:Other)|(?:Retention of [S|s]amples)|(?:Amendments? to Final Protocol))\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Guidelines found\", \"n-a\");\n });\n entityCreationService.betweenRegexes(\"(?:Deviations? (?:from|to)(?: the)? [S|s]tudy [P|p]lan)\", \"(?:Regulatory Guidelines)|(?:Other)|(?:Distribution of the report)|(?:Performing laboratory test)|(?:Distribution of the report)|(?:Retention of [S|s]amples)\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from Study Plan found\", \"n-a\");\n });\n entityCreationService.betweenStrings(\"Deviations from the study plan\", \"Regulatory Guidelines\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?>Study plan adherence)(.{1,20}deviations.{1,20} to the study plan.{0,50}\\\\.)\\\\s\", \"guideline_deviation\", EntityType.ENTITY, 1, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Guideline deviation found in text.\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeEnd(\"Deviations from the study plan\", \"validity of the study.\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.0\", \"Deviation from the study plan found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.11.1: Guideline Deviation in text\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Introduction\")\n && containsStringIgnoreCase(\"deviations from the protocol\")\n )\n then\n entityCreationService.byRegex(\"There were no deviations from the protocol.{1,100}\\\\.\\\\s\", \"guideline_deviation\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.11.1\", \"Guideline deviation found in text.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.12.0: Clinical Signs\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Clinical Signs\", \"Macroscopic Findings\") && !containsString(\"TABLE\") && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"clinical_signs\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.12.0\", \"Clinical Signs found\", \"n-a\"));\n end\n\n\nrule \"DOC.13.0: Dosages\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $section: Section(\n (anyHeadlineContainsStringIgnoreCase(\"Dosages\") || anyHeadlineContainsStringIgnoreCase(\"Study Design\"))\n && !getHeadline().containsString(\"TABLE\")\n )\n then\n entityCreationService.betweenStringsIncludeStartAndEnd(\"The animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.betweenStringsIncludeStartAndEnd(\"Animals were treated\", \".\", \"dosages\", EntityType.ENTITY, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n entityCreationService.byRegexWithLineBreaks(\"(?:\\\\.[\\\\s|\\\\n]|^.{5,20}\\\\n)([^\\\\.]{1,200}(?:animal|given|received)[^\\\\.]{1,200}dose\\\\s(?:levels?\\\\s)?(?:of|at)[^\\\\.]{1,200})(?:\\\\.[\\\\s|\\\\n|$])\", \"dosages\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.13.0\", \"Dosage found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.14.0: Mortality\"\n when\n $headline: Headline(containsString(\"Mortality\") && !containsString(\"TABLE\"))\n FileAttribute(label == \"OECD Number\", value == \"425\")\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.14.0\", \"Mortality found\", \"n-a\"));\n end\n\n\nrule \"DOC.15.0: Study Conclusion\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"404\",\"405\",\"425\",\"429\",\"436\",\"471\"))\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Conclusion\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.15.0\", \"Study Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.16.0: Weight Behavior Changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && (\n containsString(\"body weight\")\n || containsString(\"body weights\")\n || containsString(\"bodyweight\")\n || containsString(\"bodyweights\")\n )\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"weight_behavior_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.16.0\", \"Weight behavior changes found\", \"n-a\"));\n end\n\n\nrule \"DOC.17.0: Necropsy findings\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"403\",\"436\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Necropsy\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic Findings\")\n || getHeadline().containsStringIgnoreCase(\"Macroscopic examination\")\n )\n && !getHeadline().containsStringIgnoreCase(\"Table\")\n && !getHeadline().containsStringIgnoreCase(\"Appendix\")\n && !getHeadline().containsStringIgnoreCase(\"3 - MACROSCOPIC FINDINGS\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"necropsy_findings\", EntityType.ENTITY)\n .forEach( entity -> entity.apply(\"DOC.17.0\", \"Necropsy section found\", \"n-a\"));\n end\n\n\nrule \"DOC.18.0: Clinical observations\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"Clinical Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Clinical observations\")\n || anyHeadlineContainsStringIgnoreCase(\"In-life Observations\")\n || anyHeadlineContainsStringIgnoreCase(\"Postmortem Observations\")\n )\n && !anyHeadlineContainsStringIgnoreCase(\"Appendix\")\n && !anyHeadlineContainsStringIgnoreCase(\"Table\")\n && !anyHeadlineContainsStringIgnoreCase(\"Mortality\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"clinical_observations\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.18.0\", \"Clinical observations section found\", \"n-a\"));\n end\n\n\nrule \"DOC.19.0: Bodyweight changes\"\n when\n FileAttribute(label == \"OECD Number\", value == \"403\")\n $headline: Headline(containsAnyStringIgnoreCase(\"Bodyweight\", \"Bodyweights\", \"Body Weights\", \"Body Weight\"), !containsAnyStringIgnoreCase(\"Appendix\", \"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"bodyweight_changes\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.19.0\", \"Bodyweight section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.0: Study Design\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"402\",\"404\",\"405\",\"406\",\"428\",\"429\",\"438\",\"439\",\"474\",\"487\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n && !anyHeadlineContainsString(\"Preliminary screening test\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"study_design\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.20.0\", \"Study design section found\", \"n-a\"));\n end\n\n\nrule \"DOC.20.1: Study Design\"\n when\n Headline(containsStringIgnoreCase(\"Study Design\"), $sectionIdentifier: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"study_design\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.20.1\", \"Study design section found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.21.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n $parentHeadline: Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n not Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($parentHeadline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.0\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.21.1: Results and Conclusion (406, 428, 438, 439, 474 & 487)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"406\",\"428\",\"438\",\"439\",\"474\",\"487\"))\n Headline(\n containsAnyString(\"Results\", \"Conclusion\"),\n !containsAnyString(\"POSITIVE CONTROL\", \"Positive Control\", \"Evaluation\", \"Micronucleus\", \"TABLE\", \"DISCUSSION\", \"CONCLUSIONS\", \"Interpretation\",\"Viability\", \"analysis\"),\n $sectionIdentifier: getSectionIdentifier()\n )\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"results_and_conclusion\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.21.1\", \"Results and Conclusion found\", \"n-a\"));\n end\n\n\nrule \"DOC.22.0: Detailing (404 & 405)\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"404\",\"405\"))\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"Results\")\n && !getHeadline().containsStringIgnoreCase(\"Evaluation\")\n && !getHeadline().containsStringIgnoreCase(\"study\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"detailing\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.22.0\", \"Detailing found\", \"n-a\"));\n end\n\n\nrule \"DOC.23.0: Preliminary Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n ((anyHeadlineContainsString(\"Preliminary Screening Test\") && containsString(\"Clinical observations\"))\n || anyHeadlineContainsString(\"Pre-Experiment\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"preliminary_test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.23.0\", \"Preliminary Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.0: Test Results (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section((getHeadline().containsString(\"RESULTS AND DISCUSSION\") || getHeadline().containsString(\"Estimation of the proliferative response of lymph node cells\") || getHeadline().containsString(\"Results in the Main Experiment\")))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"test_results\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.24.0\", \"Test Results found\", \"n-a\"));\n end\n\n\nrule \"DOC.24.1: Test Results (429)\"\n when\n Headline(containsStringIgnoreCase(\"RESULTS AND DISCUSSION\"), $sectionIdentifierResultsAndDiscussion: getSectionIdentifier())\n $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifierResultsAndDiscussion))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"test_results\", EntityType.ENTITY)\n .forEach(entity -> {\n entity.apply(\"DOC.24.1\", \"Test Results found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.25.0: Approach used (429)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n hasEntitiesOfType(\"species\")\n && (containsStringIgnoreCase(\"animals per\") || containsStringIgnoreCase(\"animals /\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"\\\\banimals (?:per|\\\\/) .{0,15}(group)\\\\b\", \"approach_used\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.25.0\", \"Study animal approach found.\", \"n-a\");\n });\n end\n\n\nrule \"DOC.26.0: Sex\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"405\",\"429\"))\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsStringIgnoreCase(\"selection\")\n && (\n containsStringIgnoreCase(\"sex:\")\n || containsStringIgnoreCase(\"male\")\n || containsStringIgnoreCase(\"female\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"([S|s]ex:)?[\\\\w\\\\s]{0,10}\\\\b(males?|females?)\\\\b\", \"sex\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.26.0\", \"Test animal sex found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.27.0: Animal Number 405\"\n when\n FileAttribute(label == \"OECD Number\", value == \"405\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n || anyHeadlineContainsStringIgnoreCase(\"reaction\")\n )\n && !getHeadline().containsString(\"selection\")\n && (\n containsStringIgnoreCase(\"number of animals\")\n || containsStringIgnoreCase(\"no.\")\n )\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(?:.{1,10} No\\\\. )([\\\\d\\\\w\\\\-]{3,8})\", \"animal_numbers\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.27.0\", \"Number of animals found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.0: Animal Number 429\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n (\n anyHeadlineContainsStringIgnoreCase(\"animal\")\n || anyHeadlineContainsStringIgnoreCase(\"test system\")\n )\n && !getHeadline().containsString(\"selection\")\n && containsStringIgnoreCase(\"number of animals\")\n && (containsStringIgnoreCase(\"per\") || containsString(\"/\"))\n && containsStringIgnoreCase(\"group\")\n )\n then\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:)[\\\\w\\\\s]{0,10}\\\\b([\\\\d]{1,3})\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"(Number of animals per group:).{0,60}\\\\b([\\\\d]{1,3})\\\\sper group\\\\b\", \"number_of_animals\", EntityType.ENTITY,2, $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n entityCreationService.byRegexIgnoreCase(\"([\\\\d]{1,3})[\\\\w\\\\s\\\\/]{0,20}(?:treatment )?group\\\\b\", \"number_of_animals\", EntityType.ENTITY,1 , $section).forEach(entity -> {\n entity.apply(\"DOC.28.0\", \"Number of animals in group found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.28.1: No. Of animals - Fallback to appendix tables listing all individual animals for 429\"\n when\n $keyword: String() from List.of(\"Animal Number\", \"Animal No.\", \"Animal number\")\n $table: Table(containsString($keyword) && getHeadline().containsString(\"TABLE\") && getHeadline().containsString(\"Individual\"))\n FileAttribute(label == \"OECD Number\", value == \"429\")\n then\n $table.streamTableCellsWithHeader($keyword)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"animal_numbers\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.28.1\", \"Animal number found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.29.0: 4h Exposure\"\n when\n FileAttribute(label == \"OECD Number\", valueEqualsAnyOf(\"403\",\"436\"))\n $section: Section(\n (containsStringIgnoreCase(\"4 hours\") || containsStringIgnoreCase(\"four hours\"))\n )\n then\n entityCreationService.byRegexIgnoreCase(\"((?<=\\\\.\\\\s\\\\b).{1,100}(4|four) hours.*?\\\\.) \", \"4h_exposure\", EntityType.ENTITY,1, $section).forEach(entity -> {\n entity.apply(\"DOC.29.0\", \"4h exposure sentence found\", \"n-a\");\n });\n end\n\n\nrule \"DOC.30.0: Dilution of the test substance\"\n when\n FileAttribute(label == \"OECD Number\", value == \"404\")\n $section: Section(\n getHeadline().containsString(\"Formulation\")\n && containsString(\"dilution\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"dilution\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.30.0\", \"Dilution found.\", \"n-a\"));\n end\n\n\nrule \"DOC.31.0: Positive Control\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsStringIgnoreCase(\"Positive Control\")\n && !(getHeadline().containsStringIgnoreCase(\"Appendix\") || getHeadline().containsStringIgnoreCase(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"positive_control\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.31.0\", \"Positive control found.\", \"n-a\"));\n end\n\n\nrule \"DOC.32.0: Mortality Statement\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $headline: Headline(containsStringIgnoreCase(\"Mortality\") && !containsString(\"TABLE\"))\n then\n entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), \"mortality_statement\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.32.0\", \"Mortality Statement found\", \"n-a\"));\n end\n\n\nrule \"DOC.33.0: Dose Mortality\"\n when\n FileAttribute(label == \"OECD Number\", value == \"425\")\n $table: Table(\n (hasHeader(\"Mortality\") || hasHeader(\"Long Term Results\") || hasHeader(\"LongTerm Outcome\") || hasHeader(\"Long Term Outcome\") || hasHeader(\"Comments\") || hasHeader(\"Viability / Mortality\") || hasHeader(\"Viability/Mortality\"))\n &&\n (hasHeader(\"Dose [mg/kg bodyweight]\") || hasHeader(\"Dose [mg/kg body weight]\") ||hasHeader(\"Dose (mg/kg)\") || hasHeader(\"Dose levei (mg/kg)\") || hasHeader(\"Dose Level (mg/kg)\") || hasHeader(\"Dose level (mg/kg)\") || hasHeader(\"Dosage [mg/kg body weight]\"))\n )\n then\n Stream.of($table.streamTableCellsWithHeader(\"Mortality\"),\n $table.streamTableCellsWithHeader(\"Comments\"),\n $table.streamTableCellsWithHeader(\"Long Term Results\"),\n $table.streamTableCellsWithHeader(\"Long Term Outcome\"),\n $table.streamTableCellsWithHeader(\"LongTerm Outcome\"),\n $table.streamTableCellsWithHeader(\"Viability / Mortality\"),\n $table.streamTableCellsWithHeader(\"Viability/Mortality\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n\n Stream.of($table.streamTableCellsWithHeader(\"Dose [mg/kg bodyweight]\"),\n $table.streamTableCellsWithHeader(\"Dose [mg/kg body weight]\"),\n $table.streamTableCellsWithHeader(\"Dose levei (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose Level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose level (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dose (mg/kg)\"),\n $table.streamTableCellsWithHeader(\"Dosage [mg/kg body weight]\")\n ).flatMap(a -> a)\n .map(tableCell -> entityCreationService.bySemanticNode(tableCell, \"dose_mortality_dose\", EntityType.ENTITY))\n .filter(Optional::isPresent)\n .map(Optional::get)\n .forEach(entity -> {\n entity.apply(\"DOC.33.0\", \"Dose Mortality Data found.\", \"n-a\");\n insert(entity);\n });\n end\n\n\nrule \"DOC.34.0: Results (Main Study)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"429\")\n $section: Section(\n getHeadline().containsString(\"Results\")\n && getHeadline().getTextRange().length() < 20\n && !(getHeadline().containsString(\"Appendix\") || getHeadline().containsString(\"Table\"))\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"results_(main_study)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.34.0\", \"Results for main study found.\", \"n-a\"));\n end\n\n\nrule \"DOC.35.0: Doses (mg/kg bodyweight)\"\n when\n FileAttribute(label == \"OECD Number\", value == \"402\")\n $section: Section(\n anyHeadlineContainsStringIgnoreCase(\"study design\")\n )\n then\n entityCreationService.bySemanticNodeParagraphsOnly($section, \"doses_(mg_kg_bw)\", EntityType.ENTITY)\n .forEach(entity -> entity.apply(\"DOC.35.0\", \"Doses per bodyweight information found\", \"n-a\"));\n end\n\n//------------------------------------ Manual redaction rules ------------------------------------\n\n// Rule unit: MAN.0\nrule \"MAN.0.0: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $entityToBeResized: TextEntity(matchesAnnotationId($id))\n then\n manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($entityToBeResized);\n $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.0.1: Apply manual resize redaction\"\n salience 128\n when\n $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))\n $imageToBeResized: Image(id == $id)\n then\n manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);\n retract($resizeRedaction);\n update($imageToBeResized);\n update($imageToBeResized.getParent());\n end\n\n\n// Rule unit: MAN.1\nrule \"MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeRemoved: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($entityToBeRemoved);\n retract($idRemoval);\n $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));\n end\n\nrule \"MAN.1.1: Apply id removals that are valid and not in forced redactions to Image\"\n salience 128\n when\n $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageEntityToBeRemoved: Image($id == id)\n then\n $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);\n update($imageEntityToBeRemoved);\n retract($idRemoval);\n update($imageEntityToBeRemoved.getParent());\n end\n\n\n// Rule unit: MAN.2\nrule \"MAN.2.0: Apply force redaction\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToForce: TextEntity(matchesAnnotationId($id))\n then\n $entityToForce.getManualOverwrite().addChange($force);\n update($entityToForce);\n $entityToForce.getIntersectingNodes().forEach(node -> update(node));\n retract($force);\n end\n\nrule \"MAN.2.1: Apply force redaction to images\"\n salience 128\n when\n $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToForce: Image(id == $id)\n then\n $imageToForce.getManualOverwrite().addChange($force);\n update($imageToForce);\n update($imageToForce.getParent());\n retract($force);\n end\n\n\n// Rule unit: MAN.3\nrule \"MAN.3.0: Apply entity recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)\n then\n $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));\n manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);\n retract($recategorization);\n // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.\n retract($entityToBeRecategorized);\n end\n\nrule \"MAN.3.1: Apply entity recategorization of same type\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)\n then\n $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);\n retract($recategorization);\n end\n\nrule \"MAN.3.2: Apply image recategorization\"\n salience 128\n when\n $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)\n not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))\n $imageToBeRecategorized: Image($id == id)\n then\n manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);\n update($imageToBeRecategorized);\n update($imageToBeRecategorized.getParent());\n retract($recategorization);\n end\n\n\n// Rule unit: MAN.4\nrule \"MAN.4.0: Apply legal basis change\"\n salience 128\n when\n $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $imageToBeRecategorized: Image($id == id)\n then\n $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);\n end\n\nrule \"MAN.4.1: Apply legal basis change\"\n salience 128\n when\n $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)\n $entityToBeChanged: TextEntity(matchesAnnotationId($id))\n then\n $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);\n end\n\n\n\n//------------------------------------ Entity merging rules ------------------------------------\n\n// Rule unit: X.0\nrule \"X.0.0: remove Entity contained by Entity of same type\"\n salience 65\n when\n $larger: TextEntity($type: type, $entityType: entityType, active())\n $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())\n then\n $contained.remove(\"X.0.0\", \"remove Entity contained by Entity of same type\");\n retract($contained);\n end\n\n\n// Rule unit: X.2\nrule \"X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE\"\n salience 64\n when\n $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())\n $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !hasManualChanges(), active())\n then\n $entity.getIntersectingNodes().forEach(node -> update(node));\n $entity.remove(\"X.2.0\", \"remove Entity of type ENTITY when contained by FALSE_POSITIVE\");\n retract($entity)\n end\n\n\n// Rule unit: X.3\nrule \"X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\"\n salience 64\n when\n $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())\n $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.3.0\", \"remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.4\nrule \"X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\"\n salience 256\n when\n $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $entity.addEngines($recommendation.getEngines());\n $recommendation.remove(\"X.4.0\", \"remove Entity of type RECOMMENDATION when intersected by ENTITY with same type\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.5\nrule \"X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY\"\n salience 256\n when\n $entity: TextEntity(entityType == EntityType.ENTITY, active())\n $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())\n then\n $recommendation.remove(\"X.5.0\", \"remove Entity of type RECOMMENDATION when contained by ENTITY\");\n retract($recommendation);\n end\n\n\n// Rule unit: X.7\nrule \"X.7.0: remove all images\"\n salience 512\n when\n $image: Image(imageType != ImageType.OCR, !hasManualChanges())\n then\n $image.remove(\"X.7.0\", \"remove all images\");\n retract($image);\n end\n\n\n//------------------------------------ File attributes rules ------------------------------------\n\n// Rule unit: FA.1\nrule \"FA.1.0: remove duplicate FileAttributes\"\n\n salience 64\n when\n $fileAttribute: FileAttribute($label: label, $value: value)\n $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)\n then\n retract($duplicate);\n end\n\n\n// Rule unit: LDS.0\nrule \"LDS.0.0: run local dictionary search\"\n agenda-group \"LOCAL_DICTIONARY_ADDS\"\n salience -999\n when\n $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()\n then\n entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)\n .forEach(entity -> {\n Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());\n entity.addMatchedRules(matchedRules);\n });\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt deleted file mode 100644 index b5c5f258..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_Regulation_2021/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"23: Redact contact information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Contact:\")\n || text.contains(\"Alternative contact:\")\n || (text.contains(\"No:\") && text.contains(\"Fax\"))\n || (text.contains(\"Contact:\") && text.contains(\"Tel.:\"))\n || text.contains(\"European contact:\")\n ))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"24: Redact contact information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Contact:\")\n || text.contains(\"Alternative contact:\")\n || (text.contains(\"No:\") && text.contains(\"Fax\"))\n || (text.contains(\"Contact:\") && text.contains(\"Tel.:\"))\n || text.contains(\"European contact:\")\n ))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"33: Redact study director abbreviation\"\n when\n Section((searchText.contains(\"KATH\") || searchText.contains(\"BECH\") || searchText.contains(\"KML\")))\n then\n section.redactWordPartByRegEx(\"((KATH)|(BECH)|(KML)) ?(\\\\d{4})\", true, 0, 1, \"PII\", 34, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"34: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt deleted file mode 100644 index 64992263..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n/* Syngenta specific laboratory recommendation */\nrule \"0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n Section(searchText.contains(\"CT\") || searchText.contains(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"33: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"34: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt deleted file mode 100644 index df4c7799..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_GFL_v1_adress_parts/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Add CBI_author from ai 2\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.addAiEntities(\"STREET\", \"street\");\n end\n\nrule \"0: Add CBI_author from ai 3\"\n when\n Section(aiMatchesType(\"POSTAL\"))\n then\n section.addAiEntities(\"POSTAL\", \"postal\");\n end\n\nrule \"0: Add CBI_author from ai 4\"\n when\n Section(aiMatchesType(\"COUNTRY\"))\n then\n section.addAiEntities(\"COUNTRY\", \"country\");\n end\n\nrule \"0: Add CBI_author from ai 5\"\n when\n Section(aiMatchesType(\"CARDINAL\"))\n then\n section.addAiEntities(\"CARDINAL\", \"cardinal\");\n end\n\nrule \"0: Add CBI_author from ai 6\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.addAiEntities(\"CITY\", \"city\");\n end\n\nrule \"0: Add CBI_author from ai 7\"\n when\n Section(aiMatchesType(\"STATE\"))\n then\n section.addAiEntities(\"STATE\", \"state\");\n end\n\nrule \"0: Add CBI_author from ai 8\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.addAiEntities(\"ORG\", \"org\");\n end\n\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n end\n\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"12: Redact if CTL/* or BL/* was found (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (searchText.contains(\"CTL/\") || searchText.contains(\"BL/\")))\n then\n section.addHintAnnotation(\"CTL\", \"hint_only\");\n section.addHintAnnotation(\"BL\", \"hint_only\");\n end\n\n\nrule \"13: Redact if CTL/* or BL/* was found (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (searchText.contains(\"CTL/\") || searchText.contains(\"BL/\")))\n then\n section.addRedaction(\"CTL\", \"must_redact\", 13, \"Laboratory for vertebrate studies found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\" );\n section.addRedaction(\"BL\", \"must_redact\", 13, \"Laboratory for vertebrate studies found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\" );\n end\n\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"PII (Personal Identification Information) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"PII (Personal Identification Information) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\", true, 0, \"PII\", 21, \"PII (Personal Identification Information) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\", true, 0, \"PII\", 22, \"PII (Personal Identification Information) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"23: Redact contact information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"Phone No.\")\n || text.contains(\"European contact:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"24: Redact contact information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"Phone No.\")\n || text.contains(\"European contact:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"25: Redact contact information if applicant is found (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (headlineContainsWord(\"applicant\") || text.contains(\"Applicant\") || headlineContainsWord(\"Primary contact\") || headlineContainsWord(\"Alternative contact\") || text.contains(\"Telephone number:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"26: Redact contact information if applicant is found (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (headlineContainsWord(\"applicant\") || text.contains(\"Applicant\") || headlineContainsWord(\"Primary contact\") || headlineContainsWord(\"Alternative contact\") || text.contains(\"Telephone number:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"27: Redact contact information if Producer is found (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.toLowerCase().contains(\"producer of the plant protection\") || text.toLowerCase().contains(\"producer of the active substance\") || text.contains(\"Manufacturer of the active substance\") || text.contains(\"Manufacturer:\") || text.contains(\"Producer or producers of the active substance\")))\n then\n section.redactLineAfter(\"Contact:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"28: Redact contact information if Producer is found (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.toLowerCase().contains(\"producer of the plant protection\") || text.toLowerCase().contains(\"producer of the active substance\") || text.contains(\"Manufacturer of the active substance\") || text.contains(\"Manufacturer:\") || text.contains(\"Producer or producers of the active substance\")))\n then\n section.redactLineAfter(\"Contact:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"COMPLETION DATE:\") && !searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"COMPLETION DATE:\") && !searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 31, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"32: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 32, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"33: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 33, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 33, \"Performing laboratory found for non vertebrate study\");\n end\n\n\nrule \"34: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 34, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"39: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"40: Ignore dossier_redaction entries if confidential\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"41: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 41, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"42: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 42, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt deleted file mode 100644 index 261d1f82..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/EFSA_sanitisation_pre_GFL_v1/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"2: Redacted because Section contains Vertebrate\"\n when\n Section(matchesType(\"vertebrate\"))\n then\n section.redact(\"CBI_author\", 2, \"Vertebrate found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 2, \"Vertebrate found\", \"names_addresses_persons\");\n end\n\n\nrule \"3: Not Redacted because Section contains no Vertebrate\"\n when\n Section(!matchesType(\"vertebrate\"))\n then\n section.redactNot(\"CBI_author\", 3, \"No Vertebrate found\");\n section.redactNot(\"CBI_address\", 3, \"No Vertebrate found\");\n end\n\n\nrule \"4: Do not redact Names and Addresses if no redaction Indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"no_redaction_indicator\"))\n then\n section.redactNot(\"CBI_author\", 4, \"Vertebrate and No Redaction Indicator found\");\n section.redactNot(\"CBI_address\", 4, \"Vertebrate and No Redaction Indicator found\");\n end\n\n\nrule \"5: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"no_redaction_indicator\"), matchesType(\"redaction_indicator\"))\n then\n section.redact(\"CBI_author\", 5, \"Vertebrate and Redaction Indicator found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 5, \"Vertebrate and Redaction Indicator found\", \"names_addresses_persons\");\n end\n\n\nrule \"6: Do not redact Names and Addresses if no redaction Indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 6, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 6, \"Published Information found\");\n end\n\n\nrule \"7: Not redacted because Vertebrate Study = N\"\n when\n Section(rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\"))\n then\n section.redactNotCell(\"Author(s)\", 7, \"CBI_author\", true, \"Not redacted because row is not a vertebrate study\");\n section.redactNot(\"CBI_author\", 7, \"Not redacted because row is not a vertebrate study\");\n section.redactNot(\"CBI_address\", 7, \"Not redacted because row is not a vertebrate study\");\n section.highlightCell(\"Vertebrate study Y/N\", 7, \"hint_only\");\n end\n\n\nrule \"8: Redact if must redact entry is found\"\n when\n Section(matchesType(\"must_redact\"))\n then\n section.redact(\"CBI_author\", 8, \"Specification of impurity of the active substance was found.\", \"specification_impurity_active_substance\");\n section.redact(\"CBI_address\", 8, \"Specification of impurity of the active substance was found.\", \"specification_impurity_active_substance\");\n end\n\n\nrule \"9: Redact Authors and Addresses in Reference Table if it is a Vertebrate study\"\n when\n Section(rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\"))\n then\n section.redactCell(\"Author(s)\", 9, \"CBI_author\", true, \"Redacted because row is a vertebrate study\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 9, \"Redacted because row is a vertebrate study\", \"names_addresses_persons\");\n section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n end\n\n\nrule \"10: Redact sponsor company\"\n when\n Section(searchText.toLowerCase().contains(\"batches produced at\"))\n then\n section.redactIfPrecededBy(\"batches produced at\", \"CBI_sponsor\", 10, \"Redacted because it represents a sponsor company\", \"names_addresses_persons\");\n section.addHintAnnotation(\"batches produced at\", \"must_redact\");\n end\n\n\nrule \"11: Redact determination of residues\"\n when\n Section((\n searchText.toLowerCase.contains(\"determination of residues\") ||\n searchText.toLowerCase.contains(\"determination of total residues\")\n ) && (\n searchText.toLowerCase.contains(\"livestock\") ||\n searchText.toLowerCase.contains(\"live stock\") ||\n searchText.toLowerCase.contains(\"tissue\") ||\n searchText.toLowerCase.contains(\"tissues\") ||\n searchText.toLowerCase.contains(\"liver\") ||\n searchText.toLowerCase.contains(\"muscle\") ||\n searchText.toLowerCase.contains(\"bovine\") ||\n searchText.toLowerCase.contains(\"ruminant\") ||\n searchText.toLowerCase.contains(\"ruminants\")\n ))\n then\n section.redact(\"CBI_author\", 11, \"Determination of residues was found.\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 11, \"Determination of residues was found.\", \"names_addresses_persons\");\n section.addHintAnnotation(\"determination of residues\", \"must_redact\");\n section.addHintAnnotation(\"determination of total residues\", \"must_redact\");\n section.addHintAnnotation(\"livestock\", \"must_redact\");\n section.addHintAnnotation(\"live stock\", \"must_redact\");\n section.addHintAnnotation(\"tissue\", \"must_redact\");\n section.addHintAnnotation(\"tissues\", \"must_redact\");\n section.addHintAnnotation(\"liver\", \"must_redact\");\n section.addHintAnnotation(\"muscle\", \"must_redact\");\n section.addHintAnnotation(\"bovine\", \"must_redact\");\n section.addHintAnnotation(\"ruminant\", \"must_redact\");\n section.addHintAnnotation(\"ruminants\", \"must_redact\");\n end\n\n/*************************************/\n/* Syngenta specific laboratory rule */\nrule \"12: Recommend CTL/BL laboratory\"\n when\n Section(searchText.contains(\"CT\") || searchText.contains(\"BL\"))\n then\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n/*************************************/\n\nrule \"13: Redact and add recommendation for et al. author\"\n when\n Section(searchText.contains(\"et al\"))\n then\n\t\tsection.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 13, \"Author found\", \"names_addresses_persons\");\n end\n\n\nrule \"14: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n\t\tsection.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\n\nrule \"15: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n\t\tsection.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"16: Redacted PII Personal Identification Information\"\n when\n Section(matchesType(\"PII\"))\n then\n section.redact(\"PII\", 16, \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"17: Redact Emails by RegEx\"\n when\n Section(searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 17, \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"18: Redact contact information\"\n when\n Section(text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"Phone No.\")\n || text.contains(\"European contact:\"))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Email:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"European contact:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n end\n\n\nrule \"19: Redact contact information if applicant is found\"\n when\n Section(headlineContainsWord(\"applicant\") || text.contains(\"Applicant\") || headlineContainsWord(\"Primary contact\") || headlineContainsWord(\"Alternative contact\") || text.contains(\"Telephone number:\"))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Email:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"European contact:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n end\n\n\nrule \"20: Redact contact information if Producer is found\"\n when\n Section(text.toLowerCase().contains(\"producer of the plant protection\") || text.toLowerCase().contains(\"producer of the active substance\") || text.contains(\"Manufacturer of the active substance\") || text.contains(\"Manufacturer:\") || text.contains(\"Producer or producers of the active substance\"))\n then\n section.redactLineAfter(\"Contact:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n end\n\n\nrule \"21: Redact AUTHOR(S)\"\n when\n Section(searchText.contains(\"AUTHOR(S):\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 21, true, \"AUTHOR(S) was found\", \"links_producer_applicant\");\n end\n\n\nrule \"22: Redact PERFORMING LABORATORY\"\n when\n Section(searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"PII\", 22, true, \"PERFORMING LABORATORY was found\", \"links_producer_applicant\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"25: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"26: Redact signatures\"\n when\n Section(matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 26, \"Signature found\", \"names_addresses_persons\");\n end\n\n\nrule \"27: Redact Logos\"\n when\n Section(matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 27, \"Logo found\", \"names_addresses_persons\");\n end\n\nrule \"28: Redact dossier dictionary match\"\n when\n Section(matchesType(\"dossier_redaction\"))\n then\n section.redact(\"dossier_redaction\", 28, \"Specification of impurity found\", \"specification_impurity_active_substance\");\n end\n\n\nrule \"29: Ignore dossier_redaction unless confidential\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt deleted file mode 100644 index 4568fb38..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/dev/Manual_Redaction/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --- NO RULES HERE --- MANUAL REDACTION ONLY --- //\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_Regulation_2021/rules.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_GFL_v1/rules.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/EFSA_sanitisation_pre_GFL_v1/rules.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/Manual_Redaction/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/docu/Manual_Redaction/rules.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt deleted file mode 100644 index d0813054..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2021/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"23: Redact contact information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Contact:\")\n || text.contains(\"Alternative contact:\")\n || (text.contains(\"No:\") && text.contains(\"Fax\"))\n || (text.contains(\"Contact:\") && text.contains(\"Tel.:\"))\n || text.contains(\"European contact:\")\n ))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"24: Redact contact information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Contact:\")\n || text.contains(\"Alternative contact:\")\n || (text.contains(\"No:\") && text.contains(\"Fax\"))\n || (text.contains(\"Contact:\") && text.contains(\"Tel.:\"))\n || text.contains(\"European contact:\")\n ))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"25: Redact Phone and Fax by RegEx\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Ph.\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Cell\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Ph.\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Cell\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"33: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt deleted file mode 100644 index b5c5f258..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_Regulation_2022/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"23: Redact contact information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Contact:\")\n || text.contains(\"Alternative contact:\")\n || (text.contains(\"No:\") && text.contains(\"Fax\"))\n || (text.contains(\"Contact:\") && text.contains(\"Tel.:\"))\n || text.contains(\"European contact:\")\n ))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"24: Redact contact information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Contact:\")\n || text.contains(\"Alternative contact:\")\n || (text.contains(\"No:\") && text.contains(\"Fax\"))\n || (text.contains(\"Contact:\") && text.contains(\"Tel.:\"))\n || text.contains(\"European contact:\")\n ))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"33: Redact study director abbreviation\"\n when\n Section((searchText.contains(\"KATH\") || searchText.contains(\"BECH\") || searchText.contains(\"KML\")))\n then\n section.redactWordPartByRegEx(\"((KATH)|(BECH)|(KML)) ?(\\\\d{4})\", true, 0, 1, \"PII\", 34, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"34: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt deleted file mode 100644 index 64992263..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_acceptance_test/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n/* Syngenta specific laboratory recommendation */\nrule \"0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n Section(searchText.contains(\"CT\") || searchText.contains(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"33: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"34: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt deleted file mode 100644 index 64992263..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_function_tests/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n/* Syngenta specific laboratory recommendation */\nrule \"0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n Section(searchText.contains(\"CT\") || searchText.contains(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"33: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"34: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt deleted file mode 100644 index c3888e74..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_rule_test/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\nrule \"1: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\n\nrule \"2: Combine ai types CBI_author from ai\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 2, false);\n end\n\n\nrule \"3: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 3, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"4: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 4, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 5, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\n\nrule \"6: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 6, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"7: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"8: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"9: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 10, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"11: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 11, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"12: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 12, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"13: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 13, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"14: Redact addresses that start with BL or CTL\"\n when\n Section(searchText.contains(\"BL\") || searchText.contains(\"CT\"))\n then\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n\n\nrule \"15: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 16, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"17: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\n\nrule \"18: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"19: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 19, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 19, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\nrule \"30: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 30, \"PII (Personal Identification Information) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 31, \"PII (Personal Identification Information) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"32: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 32, \"PII (Personal Identification Information) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"33: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 33, \"PII (Personal Identification Information) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"34: Redact telephone numbers by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && containsRegEx(\"[+]\\\\d{1,}\", true))\n then\n section.redactByRegEx(\"((([+]\\\\d{1,3} (\\\\d{7,12})\\\\b)|([+]\\\\d{1,3}(\\\\d{3,12})\\\\b|[+]\\\\d{1,3}([ -]\\\\(?\\\\d{1,6}\\\\)?){2,4})|[+]\\\\d{1,3} ?((\\\\d{2,6}\\\\)?)([ -]\\\\d{2,6}){1,4}))(-\\\\d{1,3})?\\\\b)\", true, 1, \"PII\", 34, \"PII (Personal Identification Information) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"35: Redact telephone numbers by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && containsRegEx(\"[+]\\\\d{1,}\", true))\n then\n section.redactByRegEx(\"((([+]\\\\d{1,3} (\\\\d{7,12})\\\\b)|([+]\\\\d{1,3}(\\\\d{3,12})\\\\b|[+]\\\\d{1,3}([ -]\\\\(?\\\\d{1,6}\\\\)?){2,4})|[+]\\\\d{1,3} ?((\\\\d{2,6}\\\\)?)([ -]\\\\d{2,6}){1,4}))(-\\\\d{1,3})?\\\\b)\", true, 1, \"PII\", 35, \"PII (Personal Identification Information) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"37: Redact contact information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"Phone No.\")\n || text.contains(\"European contact:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 37, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"38: Redact contact information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"Phone No.\")\n || text.contains(\"European contact:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 38, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n\nrule \"39: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"COMPLETION DATE:\") && !searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 39, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"40: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"COMPLETION DATE:\") && !searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 40, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"41: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 41, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"42: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 42, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 43, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 43, \"Performing laboratory found for non vertebrate study\");\n end\n\n\nrule \"44: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 44, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\nrule \"50: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"51: Ignore dossier_redaction entries if confidential\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"52: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 52, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"53: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 53, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"54: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 54, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt deleted file mode 100644 index 64992263..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n/* Syngenta specific laboratory recommendation */\nrule \"0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n Section(searchText.contains(\"CT\") || searchText.contains(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"33: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"34: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt deleted file mode 100644 index df4c7799..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_GFL_v1_address_parts/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Add CBI_author from ai 2\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.addAiEntities(\"STREET\", \"street\");\n end\n\nrule \"0: Add CBI_author from ai 3\"\n when\n Section(aiMatchesType(\"POSTAL\"))\n then\n section.addAiEntities(\"POSTAL\", \"postal\");\n end\n\nrule \"0: Add CBI_author from ai 4\"\n when\n Section(aiMatchesType(\"COUNTRY\"))\n then\n section.addAiEntities(\"COUNTRY\", \"country\");\n end\n\nrule \"0: Add CBI_author from ai 5\"\n when\n Section(aiMatchesType(\"CARDINAL\"))\n then\n section.addAiEntities(\"CARDINAL\", \"cardinal\");\n end\n\nrule \"0: Add CBI_author from ai 6\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.addAiEntities(\"CITY\", \"city\");\n end\n\nrule \"0: Add CBI_author from ai 7\"\n when\n Section(aiMatchesType(\"STATE\"))\n then\n section.addAiEntities(\"STATE\", \"state\");\n end\n\nrule \"0: Add CBI_author from ai 8\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.addAiEntities(\"ORG\", \"org\");\n end\n\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n end\n\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"12: Redact if CTL/* or BL/* was found (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (searchText.contains(\"CTL/\") || searchText.contains(\"BL/\")))\n then\n section.addHintAnnotation(\"CTL\", \"hint_only\");\n section.addHintAnnotation(\"BL\", \"hint_only\");\n end\n\n\nrule \"13: Redact if CTL/* or BL/* was found (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (searchText.contains(\"CTL/\") || searchText.contains(\"BL/\")))\n then\n section.addRedaction(\"CTL\", \"must_redact\", 13, \"Laboratory for vertebrate studies found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\" );\n section.addRedaction(\"BL\", \"must_redact\", 13, \"Laboratory for vertebrate studies found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\" );\n end\n\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"PII (Personal Identification Information) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"PII (Personal Identification Information) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\", true, 0, \"PII\", 21, \"PII (Personal Identification Information) found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\", true, 0, \"PII\", 22, \"PII (Personal Identification Information) found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"23: Redact contact information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"Phone No.\")\n || text.contains(\"European contact:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 23, true, \"Contact information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"24: Redact contact information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"Phone No.\")\n || text.contains(\"European contact:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 24, true, \"Contact information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"25: Redact contact information if applicant is found (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (headlineContainsWord(\"applicant\") || text.contains(\"Applicant\") || headlineContainsWord(\"Primary contact\") || headlineContainsWord(\"Alternative contact\") || text.contains(\"Telephone number:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 25, true, \"Applicant information was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"26: Redact contact information if applicant is found (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (headlineContainsWord(\"applicant\") || text.contains(\"Applicant\") || headlineContainsWord(\"Primary contact\") || headlineContainsWord(\"Alternative contact\") || text.contains(\"Telephone number:\")))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Email:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 26, true, \"Applicant information was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"27: Redact contact information if Producer is found (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.toLowerCase().contains(\"producer of the plant protection\") || text.toLowerCase().contains(\"producer of the active substance\") || text.contains(\"Manufacturer of the active substance\") || text.contains(\"Manufacturer:\") || text.contains(\"Producer or producers of the active substance\")))\n then\n section.redactLineAfter(\"Contact:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 27, true, \"Producer was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"28: Redact contact information if Producer is found (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.toLowerCase().contains(\"producer of the plant protection\") || text.toLowerCase().contains(\"producer of the active substance\") || text.contains(\"Manufacturer of the active substance\") || text.contains(\"Manufacturer:\") || text.contains(\"Producer or producers of the active substance\")))\n then\n section.redactLineAfter(\"Contact:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Tel:\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 28, true, \"Producer was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"COMPLETION DATE:\") && !searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"COMPLETION DATE:\") && !searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 31, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"32: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"AUTHOR(S):\") && searchText.contains(\"STUDY COMPLETION DATE:\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 32, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"33: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 33, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 33, \"Performing laboratory found for non vertebrate study\");\n end\n\n\nrule \"34: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 34, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"39: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"40: Ignore dossier_redaction entries if confidential\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"41: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 41, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"42: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 42, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt deleted file mode 100644 index 261d1f82..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_sanitisation_pre_GFL_v1/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"2: Redacted because Section contains Vertebrate\"\n when\n Section(matchesType(\"vertebrate\"))\n then\n section.redact(\"CBI_author\", 2, \"Vertebrate found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 2, \"Vertebrate found\", \"names_addresses_persons\");\n end\n\n\nrule \"3: Not Redacted because Section contains no Vertebrate\"\n when\n Section(!matchesType(\"vertebrate\"))\n then\n section.redactNot(\"CBI_author\", 3, \"No Vertebrate found\");\n section.redactNot(\"CBI_address\", 3, \"No Vertebrate found\");\n end\n\n\nrule \"4: Do not redact Names and Addresses if no redaction Indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"no_redaction_indicator\"))\n then\n section.redactNot(\"CBI_author\", 4, \"Vertebrate and No Redaction Indicator found\");\n section.redactNot(\"CBI_address\", 4, \"Vertebrate and No Redaction Indicator found\");\n end\n\n\nrule \"5: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"no_redaction_indicator\"), matchesType(\"redaction_indicator\"))\n then\n section.redact(\"CBI_author\", 5, \"Vertebrate and Redaction Indicator found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 5, \"Vertebrate and Redaction Indicator found\", \"names_addresses_persons\");\n end\n\n\nrule \"6: Do not redact Names and Addresses if no redaction Indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 6, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 6, \"Published Information found\");\n end\n\n\nrule \"7: Not redacted because Vertebrate Study = N\"\n when\n Section(rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\"))\n then\n section.redactNotCell(\"Author(s)\", 7, \"CBI_author\", true, \"Not redacted because row is not a vertebrate study\");\n section.redactNot(\"CBI_author\", 7, \"Not redacted because row is not a vertebrate study\");\n section.redactNot(\"CBI_address\", 7, \"Not redacted because row is not a vertebrate study\");\n section.highlightCell(\"Vertebrate study Y/N\", 7, \"hint_only\");\n end\n\n\nrule \"8: Redact if must redact entry is found\"\n when\n Section(matchesType(\"must_redact\"))\n then\n section.redact(\"CBI_author\", 8, \"Specification of impurity of the active substance was found.\", \"specification_impurity_active_substance\");\n section.redact(\"CBI_address\", 8, \"Specification of impurity of the active substance was found.\", \"specification_impurity_active_substance\");\n end\n\n\nrule \"9: Redact Authors and Addresses in Reference Table if it is a Vertebrate study\"\n when\n Section(rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\"))\n then\n section.redactCell(\"Author(s)\", 9, \"CBI_author\", true, \"Redacted because row is a vertebrate study\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 9, \"Redacted because row is a vertebrate study\", \"names_addresses_persons\");\n section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n end\n\n\nrule \"10: Redact sponsor company\"\n when\n Section(searchText.toLowerCase().contains(\"batches produced at\"))\n then\n section.redactIfPrecededBy(\"batches produced at\", \"CBI_sponsor\", 10, \"Redacted because it represents a sponsor company\", \"names_addresses_persons\");\n section.addHintAnnotation(\"batches produced at\", \"must_redact\");\n end\n\n\nrule \"11: Redact determination of residues\"\n when\n Section((\n searchText.toLowerCase.contains(\"determination of residues\") ||\n searchText.toLowerCase.contains(\"determination of total residues\")\n ) && (\n searchText.toLowerCase.contains(\"livestock\") ||\n searchText.toLowerCase.contains(\"live stock\") ||\n searchText.toLowerCase.contains(\"tissue\") ||\n searchText.toLowerCase.contains(\"tissues\") ||\n searchText.toLowerCase.contains(\"liver\") ||\n searchText.toLowerCase.contains(\"muscle\") ||\n searchText.toLowerCase.contains(\"bovine\") ||\n searchText.toLowerCase.contains(\"ruminant\") ||\n searchText.toLowerCase.contains(\"ruminants\")\n ))\n then\n section.redact(\"CBI_author\", 11, \"Determination of residues was found.\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 11, \"Determination of residues was found.\", \"names_addresses_persons\");\n section.addHintAnnotation(\"determination of residues\", \"must_redact\");\n section.addHintAnnotation(\"determination of total residues\", \"must_redact\");\n section.addHintAnnotation(\"livestock\", \"must_redact\");\n section.addHintAnnotation(\"live stock\", \"must_redact\");\n section.addHintAnnotation(\"tissue\", \"must_redact\");\n section.addHintAnnotation(\"tissues\", \"must_redact\");\n section.addHintAnnotation(\"liver\", \"must_redact\");\n section.addHintAnnotation(\"muscle\", \"must_redact\");\n section.addHintAnnotation(\"bovine\", \"must_redact\");\n section.addHintAnnotation(\"ruminant\", \"must_redact\");\n section.addHintAnnotation(\"ruminants\", \"must_redact\");\n end\n\n/*************************************/\n/* Syngenta specific laboratory rule */\nrule \"12: Recommend CTL/BL laboratory\"\n when\n Section(searchText.contains(\"CT\") || searchText.contains(\"BL\"))\n then\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n/*************************************/\n\nrule \"13: Redact and add recommendation for et al. author\"\n when\n Section(searchText.contains(\"et al\"))\n then\n\t\tsection.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 13, \"Author found\", \"names_addresses_persons\");\n end\n\n\nrule \"14: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n\t\tsection.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\n\nrule \"15: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n\t\tsection.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"16: Redacted PII Personal Identification Information\"\n when\n Section(matchesType(\"PII\"))\n then\n section.redact(\"PII\", 16, \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"17: Redact Emails by RegEx\"\n when\n Section(searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 17, \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"18: Redact contact information\"\n when\n Section(text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"Phone No.\")\n || text.contains(\"European contact:\"))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Email:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"European contact:\", \"PII\", 18, true, \"Contact information was found\", \"links_producer_applicant\");\n end\n\n\nrule \"19: Redact contact information if applicant is found\"\n when\n Section(headlineContainsWord(\"applicant\") || text.contains(\"Applicant\") || headlineContainsWord(\"Primary contact\") || headlineContainsWord(\"Alternative contact\") || text.contains(\"Telephone number:\"))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Email:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"European contact:\", \"PII\", 19, true, \"Applicant information was found\", \"links_producer_applicant\");\n end\n\n\nrule \"20: Redact contact information if Producer is found\"\n when\n Section(text.toLowerCase().contains(\"producer of the plant protection\") || text.toLowerCase().contains(\"producer of the active substance\") || text.contains(\"Manufacturer of the active substance\") || text.contains(\"Manufacturer:\") || text.contains(\"Producer or producers of the active substance\"))\n then\n section.redactLineAfter(\"Contact:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone No.\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 20, true, \"Producer was found\", \"links_producer_applicant\");\n end\n\n\nrule \"21: Redact AUTHOR(S)\"\n when\n Section(searchText.contains(\"AUTHOR(S):\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 21, true, \"AUTHOR(S) was found\", \"links_producer_applicant\");\n end\n\n\nrule \"22: Redact PERFORMING LABORATORY\"\n when\n Section(searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"PII\", 22, true, \"PERFORMING LABORATORY was found\", \"links_producer_applicant\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"25: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"26: Redact signatures\"\n when\n Section(matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 26, \"Signature found\", \"names_addresses_persons\");\n end\n\n\nrule \"27: Redact Logos\"\n when\n Section(matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 27, \"Logo found\", \"names_addresses_persons\");\n end\n\nrule \"28: Redact dossier dictionary match\"\n when\n Section(matchesType(\"dossier_redaction\"))\n then\n section.redact(\"dossier_redaction\", 28, \"Specification of impurity found\", \"specification_impurity_active_substance\");\n end\n\n\nrule \"29: Ignore dossier_redaction unless confidential\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt deleted file mode 100644 index 446d4523..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_security_test/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n/* Syngenta specific laboratory recommendation */\nrule \"0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n Section(searchText.contains(\"CT\") || searchText.contains(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"personal_data_geolocation_article_39e3\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"33: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"34: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"personal_data_geolocation_article_39e3\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"vertebrate_study_personal_data_geolocation_article_39e2\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt deleted file mode 100644 index 64992263..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/EFSA_smoke_tests/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n/* Syngenta specific laboratory recommendation */\nrule \"0: Recommend CTL/BL laboratory that start with BL or CTL\"\n when\n Section(searchText.contains(\"CT\") || searchText.contains(\"BL\"))\n then\n /* Regular expression: ((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b)) */\n section.addRecommendationByRegEx(\"((\\\\b((([Cc]T(([1ILli\\\\/])| L|~P))|(BL))[\\\\. ]?([\\\\dA-Ziltphz~\\\\/.:!]| ?[\\\\(',][Ppi](\\\\(e)?|([\\\\(-?']\\\\/))+( ?[\\\\(\\\\/\\\\dA-Znasieg]+)?)\\\\b( ?\\\\/? ?\\\\d+)?)|(\\\\bCT[L1i]\\\\b))\", true, 0, \"CBI_address\");\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"33: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\nrule \"34: Ignore dossier_redaction entries if confidentiality is not 'confidential'\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Confidentiality\",\"confidential\") && matchesType(\"dossier_redaction\"));\n then\n section.ignore(\"dossier_redaction\");\n end\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt deleted file mode 100644 index 4568fb38..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/Manual_Redaction/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --- NO RULES HERE --- MANUAL REDACTION ONLY --- //\n" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt deleted file mode 100644 index 43dc27b6..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/PII/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redacted because Section contains Vertebrate\"\n when\n Section(matchesType(\"vertebrate\"))\n then\n section.redact(\"CBI_author\", 1, \"Vertebrate found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 1, \"Vertebrate found\", \"names_addresses_persons\");\n end\n\n\nrule \"2: Not Redacted because Section contains no Vertebrate\"\n when\n Section(!matchesType(\"vertebrate\"))\n then\n section.redactNot(\"CBI_author\", 2, \"No Vertebrate found\");\n section.redactNot(\"CBI_address\", 2, \"No Vertebrate found\");\n end\n\n\nrule \"3: Do not redact Names and Addresses if no redaction Indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"no_redaction_indicator\"))\n then\n section.redactNot(\"CBI_author\", 3, \"Vertebrate and No Redaction Indicator found\");\n section.redactNot(\"CBI_address\", 3, \"Vertebrate and No Redaction Indicator found\");\n end\n\n\nrule \"4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"no_redaction_indicator\"), matchesType(\"redaction_indicator\"))\n then\n section.redact(\"CBI_author\", 4, \"Vertebrate and Redaction Indicator found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 4, \"Vertebrate and Redaction Indicator found\", \"names_addresses_persons\");\n end\n\n\nrule \"5: Do not redact Names and Addresses if no redaction Indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 5, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 5, \"Published Information found\");\n end\n\n\nrule \"6: Not redacted because Vertebrate Study = N\"\n when\n Section(rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\"))\n then\n section.redactNotCell(\"Author(s)\", 6, \"CBI_author\", true, \"Not redacted because row is not a vertebrate study\");\n section.redactNot(\"CBI_author\", 6, \"Not redacted because row is not a vertebrate study\");\n section.redactNot(\"CBI_address\", 6, \"Not redacted because row is not a vertebrate study\");\n section.highlightCell(\"Vertebrate study Y/N\", 6, \"hint_only\");\n end\n\n\nrule \"7: Redact if must redact entry is found\"\n when\n Section(matchesType(\"must_redact\"))\n then\n section.redact(\"CBI_author\", 7, \"must_redact entry was found.\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 7, \"must_redact entry was found.\", \"names_addresses_persons\");\n end\n\n\nrule \"8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study\"\n when\n Section(rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\"))\n then\n section.redactCell(\"Author(s)\", 8, \"CBI_author\", true, \"Redacted because row is a vertebrate study\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 8, \"Redacted because row is a vertebrate study\", \"names_addresses_persons\");\n section.highlightCell(\"Vertebrate study Y/N\", 8, \"must_redact\");\n end\n\n\nrule \"9: Redact sponsor company\"\n when\n Section(searchText.toLowerCase().contains(\"batches produced at\"))\n then\n section.redactIfPrecededBy(\"batches produced at\", \"CBI_sponsor\", 9, \"Redacted because it represents a sponsor company\", \"names_addresses_persons\");\n section.addHintAnnotation(\"batches produced at\", \"must_redact\");\n end\n\n\nrule \"10: Redact determination of residues\"\n when\n Section((\n searchText.toLowerCase.contains(\"determination of residues\") ||\n searchText.toLowerCase.contains(\"determination of total residues\")\n ) && (\n searchText.toLowerCase.contains(\"livestock\") ||\n searchText.toLowerCase.contains(\"live stock\") ||\n searchText.toLowerCase.contains(\"tissue\") ||\n searchText.toLowerCase.contains(\"tissues\") ||\n searchText.toLowerCase.contains(\"liver\") ||\n searchText.toLowerCase.contains(\"muscle\") ||\n searchText.toLowerCase.contains(\"bovine\") ||\n searchText.toLowerCase.contains(\"ruminant\") ||\n searchText.toLowerCase.contains(\"ruminants\")\n ))\n then\n section.redact(\"CBI_author\", 10, \"Determination of residues was found.\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 10, \"Determination of residues was found.\", \"names_addresses_persons\");\n section.addHintAnnotation(\"determination of residues\", \"must_redact\");\n section.addHintAnnotation(\"determination of total residues\", \"must_redact\");\n section.addHintAnnotation(\"livestock\", \"must_redact\");\n section.addHintAnnotation(\"live stock\", \"must_redact\");\n section.addHintAnnotation(\"tissue\", \"must_redact\");\n section.addHintAnnotation(\"tissues\", \"must_redact\");\n section.addHintAnnotation(\"liver\", \"must_redact\");\n section.addHintAnnotation(\"muscle\", \"must_redact\");\n section.addHintAnnotation(\"bovine\", \"must_redact\");\n section.addHintAnnotation(\"ruminant\", \"must_redact\");\n section.addHintAnnotation(\"ruminants\", \"must_redact\");\n end\n\n\nrule \"11: Redact if CTL/* or BL/* was found\"\n when\n Section(searchText.contains(\"CTL/\") || searchText.contains(\"BL/\"))\n then\n section.redact(\"CBI_author\", 11, \"Laboraty for vertebrate studies found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 11, \"Laboraty for vertebrate studies found\", \"names_addresses_persons\");\n section.addHintAnnotation(\"CTL\", \"must_redact\");\n section.addHintAnnotation(\"BL\", \"must_redact\");\n end\n\n\nrule \"12: Redact and add recommendation for et al. author\"\n when\n Section(searchText.contains(\"et al\"))\n then\n\t\tsection.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 12, \"Author found\", \"names_addresses_persons\");\n end\n\n\nrule \"13: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n\t\tsection.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\n\nrule \"14: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n\t\tsection.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"14: Redacted PII Personal Identification Information\"\n when\n Section(matchesType(\"PII\"))\n then\n section.redact(\"PII\", 14, \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"15: Redact Emails by RegEx\"\n when\n Section(searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\", true, 0, \"PII\", 15, \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"16: Redact contact information\"\n when\n Section(text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"European contact:\"))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Email:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"European contact:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n end\n\n\nrule \"17: Redact contact information if applicant is found\"\n when\n Section(headlineContainsWord(\"applicant\") || text.contains(\"Applicant\") || headlineContainsWord(\"Primary contact\") || headlineContainsWord(\"Alternative contact\") || text.contains(\"Telephone number:\"))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Email:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"European contact:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n end\n\n\nrule \"18: Redact contact information if Producer is found\"\n when\n Section(text.toLowerCase().contains(\"producer of the plant protection\") || text.toLowerCase().contains(\"producer of the active substance\") || text.contains(\"Manufacturer of the active substance\") || text.contains(\"Manufacturer:\") || text.contains(\"Producer or producers of the active substance\"))\n then\n section.redactLineAfter(\"Contact:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n end\n\n\nrule \"19: Redact AUTHOR(S)\"\n when\n Section(searchText.contains(\"AUTHOR(S):\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 19, true, \"AUTHOR(S) was found\", \"links_producer_applicant\");\n end\n\n\nrule \"20: Redact PERFORMING LABORATORY\"\n when\n Section(searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"PII\", 20, true, \"PERFORMING LABORATORY was found\", \"links_producer_applicant\");\n end\n\n\nrule \"21: Redact On behalf of Sequani Ltd.:\"\n when\n Section(searchText.contains(\"On behalf of Sequani Ltd.: Name Title\"))\n then\n section.redactBetween(\"On behalf of Sequani Ltd.: Name Title\", \"On behalf of\", \"PII\", 21, false , \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"22: Redact On behalf of Syngenta Ltd.:\"\n when\n Section(searchText.contains(\"On behalf of Syngenta Ltd.: Name Title\"))\n then\n section.redactBetween(\"On behalf of Syngenta Ltd.: Name Title\", \"Study dates\", \"PII\", 22, false , \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"25: Redact Purity\"\n when\n Section(searchText.contains(\"purity\"))\n then\n\t section.redactByRegEx(\"purity ?:? (([\\\\d\\\\.]+)( .{0,4}\\\\.)? ?%)\", true, 1, \"purity\", 17, \"Purity found\", \"method_manufacture\");\n end" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt deleted file mode 100644 index fe25e1ec..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/case_insensitive/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redacted because Section contains Vertebrate\"\n when\n Section(matchesType(\"vertebrate\"))\n then\n section.redact(\"CBI_author\", 1, \"Vertebrate found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 1, \"Vertebrate found\", \"names_addresses_persons\");\n end" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt deleted file mode 100644 index 43dc27b6..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/no_redaction_indicator/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redacted because Section contains Vertebrate\"\n when\n Section(matchesType(\"vertebrate\"))\n then\n section.redact(\"CBI_author\", 1, \"Vertebrate found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 1, \"Vertebrate found\", \"names_addresses_persons\");\n end\n\n\nrule \"2: Not Redacted because Section contains no Vertebrate\"\n when\n Section(!matchesType(\"vertebrate\"))\n then\n section.redactNot(\"CBI_author\", 2, \"No Vertebrate found\");\n section.redactNot(\"CBI_address\", 2, \"No Vertebrate found\");\n end\n\n\nrule \"3: Do not redact Names and Addresses if no redaction Indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"no_redaction_indicator\"))\n then\n section.redactNot(\"CBI_author\", 3, \"Vertebrate and No Redaction Indicator found\");\n section.redactNot(\"CBI_address\", 3, \"Vertebrate and No Redaction Indicator found\");\n end\n\n\nrule \"4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"no_redaction_indicator\"), matchesType(\"redaction_indicator\"))\n then\n section.redact(\"CBI_author\", 4, \"Vertebrate and Redaction Indicator found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 4, \"Vertebrate and Redaction Indicator found\", \"names_addresses_persons\");\n end\n\n\nrule \"5: Do not redact Names and Addresses if no redaction Indicator is contained\"\n when\n Section(matchesType(\"vertebrate\"), matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 5, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 5, \"Published Information found\");\n end\n\n\nrule \"6: Not redacted because Vertebrate Study = N\"\n when\n Section(rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\"))\n then\n section.redactNotCell(\"Author(s)\", 6, \"CBI_author\", true, \"Not redacted because row is not a vertebrate study\");\n section.redactNot(\"CBI_author\", 6, \"Not redacted because row is not a vertebrate study\");\n section.redactNot(\"CBI_address\", 6, \"Not redacted because row is not a vertebrate study\");\n section.highlightCell(\"Vertebrate study Y/N\", 6, \"hint_only\");\n end\n\n\nrule \"7: Redact if must redact entry is found\"\n when\n Section(matchesType(\"must_redact\"))\n then\n section.redact(\"CBI_author\", 7, \"must_redact entry was found.\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 7, \"must_redact entry was found.\", \"names_addresses_persons\");\n end\n\n\nrule \"8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study\"\n when\n Section(rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\"))\n then\n section.redactCell(\"Author(s)\", 8, \"CBI_author\", true, \"Redacted because row is a vertebrate study\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 8, \"Redacted because row is a vertebrate study\", \"names_addresses_persons\");\n section.highlightCell(\"Vertebrate study Y/N\", 8, \"must_redact\");\n end\n\n\nrule \"9: Redact sponsor company\"\n when\n Section(searchText.toLowerCase().contains(\"batches produced at\"))\n then\n section.redactIfPrecededBy(\"batches produced at\", \"CBI_sponsor\", 9, \"Redacted because it represents a sponsor company\", \"names_addresses_persons\");\n section.addHintAnnotation(\"batches produced at\", \"must_redact\");\n end\n\n\nrule \"10: Redact determination of residues\"\n when\n Section((\n searchText.toLowerCase.contains(\"determination of residues\") ||\n searchText.toLowerCase.contains(\"determination of total residues\")\n ) && (\n searchText.toLowerCase.contains(\"livestock\") ||\n searchText.toLowerCase.contains(\"live stock\") ||\n searchText.toLowerCase.contains(\"tissue\") ||\n searchText.toLowerCase.contains(\"tissues\") ||\n searchText.toLowerCase.contains(\"liver\") ||\n searchText.toLowerCase.contains(\"muscle\") ||\n searchText.toLowerCase.contains(\"bovine\") ||\n searchText.toLowerCase.contains(\"ruminant\") ||\n searchText.toLowerCase.contains(\"ruminants\")\n ))\n then\n section.redact(\"CBI_author\", 10, \"Determination of residues was found.\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 10, \"Determination of residues was found.\", \"names_addresses_persons\");\n section.addHintAnnotation(\"determination of residues\", \"must_redact\");\n section.addHintAnnotation(\"determination of total residues\", \"must_redact\");\n section.addHintAnnotation(\"livestock\", \"must_redact\");\n section.addHintAnnotation(\"live stock\", \"must_redact\");\n section.addHintAnnotation(\"tissue\", \"must_redact\");\n section.addHintAnnotation(\"tissues\", \"must_redact\");\n section.addHintAnnotation(\"liver\", \"must_redact\");\n section.addHintAnnotation(\"muscle\", \"must_redact\");\n section.addHintAnnotation(\"bovine\", \"must_redact\");\n section.addHintAnnotation(\"ruminant\", \"must_redact\");\n section.addHintAnnotation(\"ruminants\", \"must_redact\");\n end\n\n\nrule \"11: Redact if CTL/* or BL/* was found\"\n when\n Section(searchText.contains(\"CTL/\") || searchText.contains(\"BL/\"))\n then\n section.redact(\"CBI_author\", 11, \"Laboraty for vertebrate studies found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 11, \"Laboraty for vertebrate studies found\", \"names_addresses_persons\");\n section.addHintAnnotation(\"CTL\", \"must_redact\");\n section.addHintAnnotation(\"BL\", \"must_redact\");\n end\n\n\nrule \"12: Redact and add recommendation for et al. author\"\n when\n Section(searchText.contains(\"et al\"))\n then\n\t\tsection.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 12, \"Author found\", \"names_addresses_persons\");\n end\n\n\nrule \"13: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n\t\tsection.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\n\nrule \"14: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n\t\tsection.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"14: Redacted PII Personal Identification Information\"\n when\n Section(matchesType(\"PII\"))\n then\n section.redact(\"PII\", 14, \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"15: Redact Emails by RegEx\"\n when\n Section(searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\\\.[A-Z]{2,4}\\\\b\", true, 0, \"PII\", 15, \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"16: Redact contact information\"\n when\n Section(text.contains(\"Contact point:\")\n || text.contains(\"Phone:\")\n || text.contains(\"Fax:\")\n || text.contains(\"Tel.:\")\n || text.contains(\"Tel:\")\n || text.contains(\"E-mail:\")\n || text.contains(\"Email:\")\n || text.contains(\"e-mail:\")\n || text.contains(\"E-mail address:\")\n || text.contains(\"Alternative contact:\")\n || text.contains(\"Telephone number:\")\n || text.contains(\"Telephone No:\")\n || text.contains(\"Fax number:\")\n || text.contains(\"Telephone:\")\n || text.contains(\"European contact:\"))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Email:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"European contact:\", \"PII\", 16, true, \"Contact information was found\", \"links_producer_applicant\");\n end\n\n\nrule \"17: Redact contact information if applicant is found\"\n when\n Section(headlineContainsWord(\"applicant\") || text.contains(\"Applicant\") || headlineContainsWord(\"Primary contact\") || headlineContainsWord(\"Alternative contact\") || text.contains(\"Telephone number:\"))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel.:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Email:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"e-mail:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail address:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone No:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"European contact:\", \"PII\", 17, true, \"Applicant information was found\", \"links_producer_applicant\");\n end\n\n\nrule \"18: Redact contact information if Producer is found\"\n when\n Section(text.toLowerCase().contains(\"producer of the plant protection\") || text.toLowerCase().contains(\"producer of the active substance\") || text.contains(\"Manufacturer of the active substance\") || text.contains(\"Manufacturer:\") || text.contains(\"Producer or producers of the active substance\"))\n then\n section.redactLineAfter(\"Contact:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Phone:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"E-mail:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Contact:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Fax number:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Telephone number:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactLineAfter(\"Tel:\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 18, true, \"Producer was found\", \"links_producer_applicant\");\n end\n\n\nrule \"19: Redact AUTHOR(S)\"\n when\n Section(searchText.contains(\"AUTHOR(S):\"))\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 19, true, \"AUTHOR(S) was found\", \"links_producer_applicant\");\n end\n\n\nrule \"20: Redact PERFORMING LABORATORY\"\n when\n Section(searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"PII\", 20, true, \"PERFORMING LABORATORY was found\", \"links_producer_applicant\");\n end\n\n\nrule \"21: Redact On behalf of Sequani Ltd.:\"\n when\n Section(searchText.contains(\"On behalf of Sequani Ltd.: Name Title\"))\n then\n section.redactBetween(\"On behalf of Sequani Ltd.: Name Title\", \"On behalf of\", \"PII\", 21, false , \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n\nrule \"22: Redact On behalf of Syngenta Ltd.:\"\n when\n Section(searchText.contains(\"On behalf of Syngenta Ltd.: Name Title\"))\n then\n section.redactBetween(\"On behalf of Syngenta Ltd.: Name Title\", \"Study dates\", \"PII\", 22, false , \"PII (Personal Identification Information) found\", \"links_producer_applicant\");\n end\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"25: Redact Purity\"\n when\n Section(searchText.contains(\"purity\"))\n then\n\t section.redactByRegEx(\"purity ?:? (([\\\\d\\\\.]+)( .{0,4}\\\\.)? ?%)\", true, 1, \"purity\", 17, \"Purity found\", \"method_manufacture\");\n end" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt b/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt deleted file mode 100644 index fe25e1ec..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/dossier_templates_v2/qa/vertebrate_only/rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redacted because Section contains Vertebrate\"\n when\n Section(matchesType(\"vertebrate\"))\n then\n section.redact(\"CBI_author\", 1, \"Vertebrate found\", \"names_addresses_persons\");\n section.redact(\"CBI_address\", 1, \"Vertebrate found\", \"names_addresses_persons\");\n end" \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt b/redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt deleted file mode 100644 index b5c5f258..00000000 --- a/redaction-service-v1/rules-management/src/test/resources/efsa_regulation_rules.txt +++ /dev/null @@ -1 +0,0 @@ -"package drools\n\nimport com.iqser.red.service.redaction.v1.server.redaction.model.Section\n\nglobal Section section\n\n\n// --------------------------------------- AI rules -------------------------------------------------------------------\n\nrule \"0: Add CBI_author from ai\"\n when\n Section(aiMatchesType(\"CBI_author\"))\n then\n section.addAiEntities(\"CBI_author\", \"CBI_author\");\n end\n\nrule \"0: Combine address parts from ai to CBI_address (org is mandatory)\"\n when\n Section(aiMatchesType(\"ORG\"))\n then\n section.combineAiTypes(\"ORG\", \"STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (street is mandatory)\"\n when\n Section(aiMatchesType(\"STREET\"))\n then\n section.combineAiTypes(\"STREET\", \"ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE\", 20, \"CBI_address\", 3, false);\n end\n\nrule \"0: Combine address parts from ai to CBI_address (city is mandatory)\"\n when\n Section(aiMatchesType(\"CITY\"))\n then\n section.combineAiTypes(\"CITY\", \"ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE\", 20, \"CBI_address\", 3, false);\n end\n\n\n// --------------------------------------- CBI rules -------------------------------------------------------------------\n\nrule \"1: Redact CBI Authors (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 1, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"2: Redact CBI Authors (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_author\"))\n then\n section.redact(\"CBI_author\", 2, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"3: Redact not CBI Address (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redactNot(\"CBI_address\", 3, \"Address found for non vertebrate study\");\n section.ignoreRecommendations(\"CBI_address\");\n end\n\nrule \"4: Redact CBI Address (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"CBI_address\"))\n then\n section.redact(\"CBI_address\", 4, \"Address found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"5: Do not redact genitive CBI_author\"\n when\n Section(matchesType(\"CBI_author\"))\n then\n section.expandToFalsePositiveByRegEx(\"CBI_author\", \"['’’'ʼˈ´`‘′ʻ’']s\", false, 0);\n end\n\n\nrule \"6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 6, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author(s)\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author(s)\", 7, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"8: Redact Author cells in Tables with Author header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 8, \"CBI_author\", false, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"9: Redact Author cells in Tables with Author header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && hasTableHeader(\"Author\") && !hasTableHeader(\"Vertebrate study Y/N\"))\n then\n section.redactCell(\"Author\", 9, \"CBI_author\", false, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 10, \"CBI_author\", true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", \"Yes\") || rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\")))\n then\n section.redactCell(\"Author(s)\", 11, \"CBI_author\", true, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"14: Redact and add recommendation for et al. author (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 14, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"15: Redact and add recommendation for et al. author (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"et al\"))\n then\n section.redactAndRecommendByRegEx(\"\\\\b([A-ZÄÖÜ][^\\\\s\\\\.,]+( [A-ZÄÖÜ]{1,2}\\\\.?)?( ?[A-ZÄÖÜ]\\\\.?)?) et al\\\\.?\", false, 1, \"CBI_author\", 15, \"Author found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"16: Add recommendation for Addresses in Test Organism sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species:\") && searchText.contains(\"Source:\"))\n then\n section.recommendLineAfter(\"Source:\", \"CBI_address\");\n end\n\nrule \"17: Add recommendation for Addresses in Test Animals sections\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"Species\") && searchText.contains(\"Source\"))\n then\n section.recommendLineAfter(\"Source\", \"CBI_address\");\n end\n\n\nrule \"18: Do not redact Names and Addresses if Published Information found\"\n when\n Section(matchesType(\"published_information\"))\n then\n section.redactNotAndReference(\"CBI_author\",\"published_information\", 18, \"Published Information found\");\n section.redactNotAndReference(\"CBI_address\",\"published_information\", 18, \"Published Information found\");\n end\n\n\n// --------------------------------------- PII rules -------------------------------------------------------------------\n\n\nrule \"19: Redacted PII Personal Identification Information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 19, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"20: Redacted PII Personal Identification Information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesType(\"PII\"))\n then\n section.redact(\"PII\", 20, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"21: Redact Emails by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 21, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"22: Redact Emails by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && searchText.contains(\"@\"))\n then\n section.redactByRegEx(\"\\\\b([A-Za-z0-9._%+\\\\-]+@[A-Za-z0-9.\\\\-]+\\\\.[A-Za-z\\\\-]{1,23}[A-Za-z])\\\\b\", true, 1, \"PII\", 22, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"23: Redact contact information (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Contact:\")\n || text.contains(\"Alternative contact:\")\n || (text.contains(\"No:\") && text.contains(\"Fax\"))\n || (text.contains(\"Contact:\") && text.contains(\"Tel.:\"))\n || text.contains(\"European contact:\")\n ))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 23, true, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"24: Redact contact information (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (text.contains(\"Contact point:\")\n || text.contains(\"Contact:\")\n || text.contains(\"Alternative contact:\")\n || (text.contains(\"No:\") && text.contains(\"Fax\"))\n || (text.contains(\"Contact:\") && text.contains(\"Tel.:\"))\n || text.contains(\"European contact:\")\n ))\n then\n section.redactLineAfter(\"Contact point:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"Alternative contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"No:\", \"Fax\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactBetween(\"Contact:\", \"Tel.:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n section.redactLineAfter(\"European contact:\", \"PII\", 24, true, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"25: Redact Phone and Fax by RegEx (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 25, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"26: Redact Phone and Fax by RegEx (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && (\n text.contains(\"Contact\")\n || text.contains(\"Telephone\")\n || text.contains(\"Phone\")\n || text.contains(\"Fax\")\n || text.contains(\"Tel\")\n || text.contains(\"Ter\")\n || text.contains(\"Mobile\")\n || text.contains(\"Fel\")\n || text.contains(\"Fer\")\n ))\n then\n section.redactByRegEx(\"\\\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\\\s]{0,10}[:.\\\\s]{0,3}([\\\\+\\\\d\\\\(][\\\\s\\\\d\\\\(\\\\)\\\\-\\\\/\\\\.]{4,100}\\\\d)\\\\b\", true, 2, \"PII\", 26, \"Personal information found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"27: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 27, true, \"Author found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"28: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"COMPLETION DATE:\")\n && !searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"COMPLETION DATE:\", \"PII\", 28, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"29: Redact AUTHOR(S) (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 29, true, \"AUTHOR(S) was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"30: Redact AUTHOR(S) (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"AUTHOR(S):\")\n && searchText.contains(\"STUDY COMPLETION DATE:\")\n )\n then\n section.redactLinesBetween(\"AUTHOR(S):\", \"STUDY COMPLETION DATE:\", \"PII\", 30, true, \"AUTHOR(S) was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"31: Redact PERFORMING LABORATORY (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\")\n )\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 31, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n section.redactNot(\"CBI_address\", 31, \"Performing laboratory found for non vertebrate study\");\n end\n\nrule \"32: Redact PERFORMING LABORATORY (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\")\n && searchText.contains(\"PERFORMING LABORATORY:\"))\n then\n section.redactBetween(\"PERFORMING LABORATORY:\", \"LABORATORY PROJECT ID:\", \"CBI_address\", 32, true, \"PERFORMING LABORATORY was found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\nrule \"33: Redact study director abbreviation\"\n when\n Section((searchText.contains(\"KATH\") || searchText.contains(\"BECH\") || searchText.contains(\"KML\")))\n then\n section.redactWordPartByRegEx(\"((KATH)|(BECH)|(KML)) ?(\\\\d{4})\", true, 0, 1, \"PII\", 34, \"Personal information found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\n\n// --------------------------------------- other rules -------------------------------------------------------------------\n\nrule \"34: Purity Hint\"\n when\n Section(searchText.toLowerCase().contains(\"purity\"))\n then\n\t section.addHintAnnotationByRegEx(\"(purity ?( of|\\\\(.{1,20}\\\\))?( ?:)?) .{0,5}[\\\\d\\\\.]+( .{0,4}\\\\.)? ?%\", true, 1, \"hint_only\");\n end\n\n\n\nrule \"35: Redact signatures (Non vertebrate study)\"\n when\n Section(!fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 35, \"Signature found\", \"Article 39(e)(3) of Regulation (EC) No 178/2002\");\n end\n\nrule \"36: Redact signatures (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"signature\"))\n then\n section.redactImage(\"signature\", 36, \"Signature found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n\n\nrule \"43: Redact Logos (Vertebrate study)\"\n when\n Section(fileAttributeByLabelEqualsIgnoreCase(\"Vertebrate Study\",\"Yes\") && matchesImageType(\"logo\"))\n then\n section.redactImage(\"logo\", 43, \"Logo found\", \"Article 39(e)(2) of Regulation (EC) No 178/2002\");\n end\n" \ No newline at end of file