From f3bfe95ea7ddb55323dd58181133849d6fb67926 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Tue, 7 Mar 2023 14:20:27 +0100 Subject: [PATCH] RED-6093: Prototype document structure *refactored File Structure *started refactor of original rules --- .../document/data/AtomicTextBlockData.java | 21 + .../v1/server/document/data/DocumentData.java | 19 + .../v1/server/document/data/PageData.java | 20 + .../document/data/TableOfContentsData.java | 52 +++ .../v1/server/document/graph/Boundary.java | 6 +- .../server/document/graph/DocumentGraph.java | 26 +- .../document/graph/TableOfContents.java | 19 +- ...cumentNode.java => DocumentGraphNode.java} | 43 +- .../document/graph/nodes/EntityNode.java | 29 +- .../document/graph/nodes/FooterNode.java | 71 ---- .../document/graph/nodes/HeaderNode.java | 72 ---- .../server/document/graph/nodes/NodeType.java | 28 ++ .../server/document/graph/nodes/PageNode.java | 61 +-- .../document/graph/nodes/ParagraphNode.java | 14 +- .../document/graph/nodes/SectionNode.java | 17 +- .../document/graph/nodes/TableCellNode.java | 8 +- .../document/graph/nodes/TableNode.java | 7 +- .../graph/textblock/AtomicTextBlock.java | 30 +- .../textblock/ConcatenatedTextBlock.java | 2 +- .../document/graph/textblock/TextBlock.java | 14 +- .../document/services/DocumentDataMapper.java | 98 +++++ .../services/DocumentGraphFactory.java | 75 ++-- .../services/DocumentGraphMapper.java | 171 ++++++++ .../document/services/RegexMatcher.java | 2 +- .../server/AbstractTestWithDictionaries.java | 6 +- .../server/DocumentGraphIntegrationTest.java | 15 +- .../v1/server/DocumentGraphMappingTest.java | 55 +++ .../src/test/resources/drools/allAuthors.drl | 126 +++--- .../test/resources/drools/entity_rules.drl | 4 +- .../src/test/resources/drools/headlines.drl | 2 +- .../src/test/resources/drools/prod_rules.drl | 383 ++++++++++++++++++ .../src/test/resources/drools/rules.drl | 63 +-- .../src/test/resources/drools/rules2.drl | 1 + .../src/test/resources/drools/testRules.drl | 124 +++--- .../src/test/resources/drools/test_rules.drl | 12 +- .../EFSA_sanitisation_GFL_v1/rules.drl | 242 +++++------ 36 files changed, 1329 insertions(+), 609 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/{DocumentNode.java => DocumentGraphNode.java} (85%) delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentDataMapper.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphMappingTest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_rules.drl create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java new file mode 100644 index 00000000..f9103d2c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.redaction.v1.server.document.data; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class AtomicTextBlockData { + Long id; + String searchText; + int start; + int end; + int[] lineBreaks; + int[] stringIdxToPositionIdx; + float[][] positions; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java new file mode 100644 index 00000000..54a39c5f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.server.document.data; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DocumentData { + List pages; + List atomicTextBlocks; + TableOfContentsData tableOfContents; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java new file mode 100644 index 00000000..a00747bb --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.document.data; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class PageData { + int number; + int height; + int width; + + Long header; + Long footer; +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java new file mode 100644 index 00000000..17273696 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java @@ -0,0 +1,52 @@ +package com.iqser.red.service.redaction.v1.server.document.data; + +import static java.lang.String.format; + +import java.util.Arrays; +import java.util.List; + +import javax.management.openmbean.InvalidKeyException; + +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class TableOfContentsData { + + List entries; + + + public EntryData get(String tocId) { + + List ids = getIds(tocId); + if (ids.size() < 1) { + throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", tocId)); + } + EntryData entry = entries.get(ids.get(0)); + for (int id : ids.subList(1, ids.size())) { + entry = entry.subEntries().get(id); + } + return entry; + } + + + private static List getIds(String idsAsString) { + + return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList(); + } + + + @Builder + public record EntryData(String tocId, List subEntries, NodeType type, Long atomicTextBlock, Long page, int numberOnPage) { + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java index f59b8b4d..01121463 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java @@ -18,7 +18,9 @@ public class Boundary { this.end = end; } + public int length() { + return end - start; } @@ -76,9 +78,11 @@ public class Boundary { return contains(boundary.start()) || contains(boundary.end()); } + @Override public String toString() { - return format("Range [%d|%d)", start, end); + + return format("Boundary [%d|%d)", start, end); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java index d08b9cac..93eca932 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java @@ -7,10 +7,11 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; @@ -40,7 +41,17 @@ public class DocumentGraph { public ConcatenatedTextBlock buildTextBlock() { - return streamAllNodes().filter(DocumentNode::isTerminal).map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); + return streamAtomicTextBlocksInOrder().collect(new TextBlockCollector()); + } + + + public Stream streamAtomicTextBlocksInOrder() { + + return Stream.concat(// + streamAllNodes().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getAtomicTextBlock),// + Stream.concat(// + pages.stream().map(PageNode::getHeader),// + pages.stream().map(PageNode::getFooter))); } @@ -64,20 +75,15 @@ public class DocumentGraph { } - public Set getEntities() { - return streamAllNodes().filter(DocumentNode::isTerminal).map(DocumentNode::getEntities).flatMap(List::stream).collect(Collectors.toSet()); + return streamAllNodes().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getEntities).flatMap(List::stream).collect(Collectors.toSet()); } - private Stream streamAllNodes() { + private Stream streamAllNodes() { - return Stream.concat(// - tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node), // - Stream.concat(// - pages.stream().map(PageNode::getHeader), // - pages.stream().map(PageNode::getFooter))); // + return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java index 9a0dece8..ec37e77a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java @@ -11,18 +11,14 @@ import java.util.stream.Stream; import javax.management.openmbean.InvalidKeyException; import com.google.common.hash.Hashing; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import lombok.Data; @Data public class TableOfContents { - public static final String SECTION = "Sec"; - public static final String HEADLINE = "H"; - public static final String PARAGRAPH = "Par"; - public static final String TABLE = "Tab"; - List entries; @@ -32,19 +28,19 @@ public class TableOfContents { } - public String createNewEntryAndReturnId(String keyword, String summary, DocumentNode node) { + public String createNewEntryAndReturnId(NodeType nodeType, String summary, DocumentGraphNode node) { String id = String.format("%d", entries.size()); - entries.add(new Entry(keyword, id, summary, new LinkedList<>(), node)); + entries.add(new Entry(nodeType, id, summary, new LinkedList<>(), node)); return id; } - public String createNewChildEntryAndReturnId(String parentId, String keyword, String summary, DocumentNode node) { + public String createNewChildEntryAndReturnId(String parentId, NodeType nodeType, String summary, DocumentGraphNode node) { Entry parent = getEntryById(parentId); String childId = parentId + String.format(".%d", parent.children().size()); - parent.children().add(new Entry(keyword, childId, summary, new LinkedList<>(), node)); + parent.children().add(new Entry(nodeType, childId, summary, new LinkedList<>(), node)); return childId; } @@ -100,7 +96,7 @@ public class TableOfContents { } - public record Entry(String type, String id, String summary, List children, DocumentNode node) { + public record Entry(NodeType type, String id, String summary, List children, DocumentGraphNode node) { @Override public String toString() { @@ -108,6 +104,7 @@ public class TableOfContents { return id + ": " + type + ".: " + summary; } + @Override public int hashCode() { return Hashing.murmur3_32_fixed().hashString(type + id + summary + children.hashCode(), StandardCharsets.UTF_8).hashCode(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentGraphNode.java similarity index 85% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentNode.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentGraphNode.java index 004b9a1d..14877d81 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentGraphNode.java @@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentUtility.enrichEntity; import static java.lang.String.format; -import java.util.List; +import java.util.Set; import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; @@ -12,7 +12,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBl import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; -public interface DocumentNode { +public interface DocumentGraphNode { /** * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity. @@ -24,12 +24,12 @@ public interface DocumentNode { /** - * Any Node maintains its own List of Entities. - * This List contains all Entities, whose first index is located in any of the AtomicTextBlocks underneath this Node. + * Any Node maintains its own Set of Entities. + * This Set contains all Entities, whose first index is located in any of the AtomicTextBlocks underneath this Node. * - * @return List of all Entities associated with this Node + * @return Set of all Entities associated with this Node */ - List getEntities(); + Set getEntities(); /** @@ -49,10 +49,27 @@ public interface DocumentNode { * * @return Node that represents the Parent or null, if no parent is present. */ - DocumentNode getParent(); + DocumentGraphNode getParent(); - Stream streamAllSubNodes(); + Stream streamAllSubNodes(); + + + /** + * Each AtomicTextBlock has a number assigned per page, this returns the number of the first AtomicTextBlock underneath this node + * + * @return Integer representing the number on the page + */ + Integer getNumberOnPage(); + + + /** + * + * @return the fist headline whent traversing the tree upwards + */ + default CharSequence getHeadline() { + return getParent().getHeadline(); + } /** @@ -165,7 +182,7 @@ public interface DocumentNode { private void setFields(EntityNode entity, AtomicTextBlock atomicTextBlock) { - if (atomicTextBlock.containsRange(entity.getBoundary())) { + if (atomicTextBlock.containsBoundary(entity.getBoundary())) { enrichEntity(entity, atomicTextBlock); } else { this.setFieldsFromParents(this, entity); @@ -175,7 +192,7 @@ public interface DocumentNode { private void addEntityToParents(EntityNode entity) { - DocumentNode node = this; + DocumentGraphNode node = this; while (node.hasParent()) { node = node.getParent(); node.getEntities().add(entity); @@ -184,12 +201,12 @@ public interface DocumentNode { } - private void setFieldsFromParents(DocumentNode node, EntityNode entity) { + private void setFieldsFromParents(DocumentGraphNode node, EntityNode entity) { if (node.hasParent()) { - DocumentNode parent = node.getParent(); + DocumentGraphNode parent = node.getParent(); TextBlock textBlock = parent.buildTextBlock(); - if (textBlock.containsRange(entity.getBoundary())) { + if (textBlock.containsBoundary(entity.getBoundary())) { enrichEntity(entity, textBlock); return; } else { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java index ca99c08e..2c279ae0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java @@ -7,7 +7,9 @@ import java.util.List; import java.util.Set; import com.google.common.hash.Hashing; +import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import lombok.AccessLevel; @@ -34,24 +36,41 @@ public class EntityNode { EntityType entityType; @Builder.Default - boolean redact = false; + boolean redaction = false; @Builder.Default - boolean recommend = false; + boolean falsePositive = false; @Builder.Default boolean removed = false; + @Builder.Default + boolean ignored = false; + @Builder.Default + boolean resized = false; + @Builder.Default + boolean skipRemoveEntitiesContainedInLarger = false; + @Builder.Default + boolean isDictionaryEntry = false; + @Builder.Default + Set engines = new HashSet<>(); + @Builder.Default + Set references = new HashSet<>(); + @Builder.Default + int matchedRule = -1; + @Builder.Default + String redactionReason = ""; + @Builder.Default + String legalBasis = ""; // inferrable from graph - int uniqueId; String value; CharSequence textBefore; CharSequence textAfter; PageNode page; List positions; @Builder.Default - Set containingNodes = new HashSet<>(); + Set containingNodes = new HashSet<>(); - public void addContainingNode(DocumentNode containingNode) { + public void addContainingNode(DocumentGraphNode containingNode) { containingNodes.add(containingNode); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java deleted file mode 100644 index ba360307..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; - -import java.util.LinkedList; -import java.util.List; -import java.util.stream.Stream; - -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class FooterNode implements DocumentNode { - - @EqualsAndHashCode.Exclude - PageNode page; - AtomicTextBlock atomicTextBlock; - - @Builder.Default - @EqualsAndHashCode.Exclude - List entities = new LinkedList<>(); - - - @Override - public AtomicTextBlock buildTextBlock() { - - return atomicTextBlock; - } - - @Override - public DocumentNode getParent() { - - throw new UnsupportedOperationException("Footer has no Parent, use getPage() instead"); - } - - - @Override - public Stream streamAllSubNodes() { - - return Stream.of(this); - } - - - @Override - public boolean hasParent() { - - return false; - } - - - @Override - public boolean isTerminal() { - - return true; - } - - - @Override - public String toString() { - - return atomicTextBlock.toString(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java deleted file mode 100644 index 0c0d7ea8..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; - -import java.util.LinkedList; -import java.util.List; -import java.util.stream.Stream; - -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class HeaderNode implements DocumentNode { - - @EqualsAndHashCode.Exclude - PageNode page; - AtomicTextBlock atomicTextBlock; - - @Builder.Default - @EqualsAndHashCode.Exclude - List entities = new LinkedList<>(); - - - @Override - public AtomicTextBlock buildTextBlock() { - - return atomicTextBlock; - } - - - @Override - public DocumentNode getParent() { - - throw new UnsupportedOperationException("Header has no Parent, use getPage() instead"); - } - - - @Override - public Stream streamAllSubNodes() { - - return Stream.of(this); - } - - - @Override - public boolean hasParent() { - - return false; - } - - - @Override - public boolean isTerminal() { - - return true; - } - - - @Override - public String toString() { - - return atomicTextBlock.toString(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java new file mode 100644 index 00000000..631749ef --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +public enum NodeType { + SECTION { + public String toString() { + + return "Section"; + } + }, + PARAGRAPH { + public String toString() { + + return "Paragraph"; + } + }, + TABLE { + public String toString() { + + return "Table"; + } + }, + TABLE_CELL { + public String toString() { + + return "Cell"; + } + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java index e575dc43..c1b91da8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java @@ -1,11 +1,13 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; -import java.util.LinkedList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -18,42 +20,24 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class PageNode implements DocumentNode { +public class PageNode implements DocumentGraphNode{ Integer number; Integer height; Integer width; - List mainBody; - HeaderNode header; - FooterNode footer; + List mainBody; + AtomicTextBlock header; + AtomicTextBlock footer; @Builder.Default @EqualsAndHashCode.Exclude - List entities = new LinkedList<>(); + Set entities = new HashSet<>(); + - @Override public ConcatenatedTextBlock buildTextBlock() { - return mainBody.stream().filter(DocumentNode::isTerminal).map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); - } - - - @Override - public DocumentNode getParent() { - - throw new UnsupportedOperationException("A Page has no parent!"); - } - - - @Override - public Stream streamAllSubNodes() { - - return Stream.concat(// - mainBody.stream(), // - Stream.concat(// - Stream.of(header), // - Stream.of(footer))); // + return mainBody.stream().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getAtomicTextBlock).collect(new TextBlockCollector()); } @@ -64,6 +48,13 @@ public class PageNode implements DocumentNode { } + @Override + public DocumentGraphNode getParent() { + + return null; + } + + @Override public boolean hasParent() { @@ -71,10 +62,24 @@ public class PageNode implements DocumentNode { } + @Override + public Stream streamAllSubNodes() { + + return mainBody.stream(); + } + + + @Override + public Integer getNumberOnPage() { + + return 0; + } + + @Override public String toString() { - return number.toString(); + return header.getSearchText() + buildTextBlock().toString() + footer.getSearchText(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java index c6f243ae..258a64f9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java @@ -1,7 +1,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; -import java.util.LinkedList; -import java.util.List; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; @@ -17,10 +17,9 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class ParagraphNode implements DocumentNode { +public class ParagraphNode implements DocumentGraphNode { String tocId; - Integer id; Integer numberOnPage; Integer numberInSection; @@ -32,7 +31,7 @@ public class ParagraphNode implements DocumentNode { @Builder.Default @EqualsAndHashCode.Exclude - List entities = new LinkedList<>(); + Set entities = new HashSet<>(); @Override @@ -43,7 +42,7 @@ public class ParagraphNode implements DocumentNode { @Override - public DocumentNode getParent() { + public DocumentGraphNode getParent() { return parentSection; } @@ -63,8 +62,9 @@ public class ParagraphNode implements DocumentNode { } @Override - public Stream streamAllSubNodes() { + public Stream streamAllSubNodes() { return Stream.of(this); } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java index dec1b62e..7f04e9c3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java @@ -1,7 +1,8 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; -import java.util.LinkedList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; @@ -22,14 +23,14 @@ import lombok.extern.slf4j.Slf4j; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class SectionNode implements DocumentNode { +public class SectionNode implements DocumentGraphNode { String tocId; - Integer id; + Integer numberOnPage; @EqualsAndHashCode.Exclude TableOfContents tableOfContents; @EqualsAndHashCode.Exclude - DocumentNode parentSection; + DocumentGraphNode parentSection; AtomicTextBlock headline; @@ -42,7 +43,7 @@ public class SectionNode implements DocumentNode { @Builder.Default @EqualsAndHashCode.Exclude - List entities = new LinkedList<>(); + Set entities = new HashSet<>(); @Override @@ -55,12 +56,12 @@ public class SectionNode implements DocumentNode { @Override public ConcatenatedTextBlock buildTextBlock() { - return streamAllSubNodes().map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); + return streamAllSubNodes().map(DocumentGraphNode::getAtomicTextBlock).collect(new TextBlockCollector()); } @Override - public DocumentNode getParent() { + public DocumentGraphNode getParent() { if (hasParent()) { return parentSection; @@ -71,7 +72,7 @@ public class SectionNode implements DocumentNode { @Override - public Stream streamAllSubNodes() { + public Stream streamAllSubNodes() { return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java index 55553077..244731bf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java @@ -17,10 +17,11 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class TableCellNode implements DocumentNode { +public class TableCellNode implements DocumentGraphNode { @EqualsAndHashCode.Exclude TableNode parentTable; + Integer numberOnPage; AtomicTextBlock atomicTextBlock; PageNode page; @@ -37,7 +38,7 @@ public class TableCellNode implements DocumentNode { @Override - public DocumentNode getParent() { + public DocumentGraphNode getParent() { return parentTable; } @@ -50,8 +51,9 @@ public class TableCellNode implements DocumentNode { } @Override - public Stream streamAllSubNodes() { + public Stream streamAllSubNodes() { return Stream.of(this); } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java index b03b5ee5..bfb99062 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java @@ -20,12 +20,13 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class TableNode implements DocumentNode { +public class TableNode implements DocumentGraphNode { Integer id; String tocId; Integer numberOfRows; Integer numberOfCols; + Integer numberOnPage; List tableHeaders; List> tableCells; TableOfContents tableOfContents; @@ -65,14 +66,14 @@ public class TableNode implements DocumentNode { } @Override - public Stream streamAllSubNodes() { + public Stream streamAllSubNodes() { return streamTableCells().map(Function.identity()); } @Override - public DocumentNode getParent() { + public DocumentGraphNode getParent() { return parentSection; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java index d846081d..7a8ceaf9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java @@ -6,7 +6,7 @@ import java.awt.geom.Rectangle2D; import java.util.List; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -21,7 +21,7 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class AtomicTextBlock implements TextBlock { - Integer id; + Long id; //string coordinates Boundary boundary; @@ -33,7 +33,7 @@ public class AtomicTextBlock implements TextBlock { List positions; @EqualsAndHashCode.Exclude - DocumentNode parent; + DocumentGraphNode parent; public int indexOf(String searchTerm) { @@ -80,21 +80,19 @@ public class AtomicTextBlock implements TextBlock { } - public List getPositions(Boundary range) { + public List getPositions(Boundary boundary) { - if (!containsRange(range)) { - throw new IndexOutOfBoundsException(format("String index Range [%d|%d) is out of boundary for String index [%d|%d)", - range.start(), - range.end(), - range.start(), - range.end())); + if (!containsBoundary(boundary)) { + throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", + boundary, + this.boundary)); } - if (range.end() == this.boundary.end()) { - return positions.subList(stringIdxToPositionIdx.get(range.start() - this.boundary.start()), positions.size()); + if (boundary.end() == this.boundary.end()) { + return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), positions.size()); } - return positions.subList(stringIdxToPositionIdx.get(range.start() - this.boundary.start()), stringIdxToPositionIdx.get(range.end() - this.boundary.start())); + return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), stringIdxToPositionIdx.get(boundary.end() - this.boundary.start())); } @@ -119,12 +117,6 @@ public class AtomicTextBlock implements TextBlock { } - public String getFirstLine() { - - return searchText.substring(0, getNextLinebreak(0) - boundary.start()); - } - - @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java index 20f36a2b..8e1d68d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java @@ -51,7 +51,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier atomicTextBlockData = documentGraph.streamAtomicTextBlocksInOrder().map(this::toAtomicTextBlockData).toList(); + List pageData = documentGraph.getPages().stream().map(this::toPageData).toList(); + TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents()); + return DocumentData.builder().atomicTextBlocks(atomicTextBlockData).pages(pageData).tableOfContents(tableOfContentsData).build(); + } + + + private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) { + + return new TableOfContentsData(tableOfContents.getEntries().stream().map(this::toEntryData).toList()); + } + + + private TableOfContentsData.EntryData toEntryData(TableOfContents.Entry entry) { + + return TableOfContentsData.EntryData.builder() + .tocId(entry.id()) + .subEntries(entry.children().stream().map(this::toEntryData).toList()) + .type(entry.type()) + .atomicTextBlock(entry.node().isTerminal() ? entry.node().getAtomicTextBlock().getId() : -1L) + .page(Long.valueOf(entry.node().getPage().getNumber())) + .numberOnPage(entry.node().getNumberOnPage()) + .build(); + } + + + private PageData toPageData(PageNode pageNode) { + + return PageData.builder() + .height(pageNode.getHeight()) + .width(pageNode.getWidth()) + .number(pageNode.getNumber()) + .footer(pageNode.getFooter().getId()) + .header(pageNode.getHeader().getId()) + .build(); + } + + + private AtomicTextBlockData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) { + + return AtomicTextBlockData.builder() + .id(atomicTextBlock.getId()) + .searchText(atomicTextBlock.getSearchText()) + .start(atomicTextBlock.getBoundary().start()) + .end(atomicTextBlock.getBoundary().end()) + .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks())) + .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx())) + .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions())) + .build(); + } + + + private float[][] toPrimitiveFloatMatrix(List positions) { + + float[][] positionMatrix = new float[positions.size()][]; + for (int i = 0; i < positions.size(); i++) { + float[] singlePositions = new float[4]; + singlePositions[0] = (float) positions.get(i).getMinX(); + singlePositions[1] = (float) positions.get(i).getMinY(); + singlePositions[2] = (float) positions.get(i).getWidth(); + singlePositions[3] = (float) positions.get(i).getHeight(); + positionMatrix[i] = singlePositions; + } + return positionMatrix; + } + + + private int[] toPrimitiveIntArray(List list) { + + int[] array = new int[list.size()]; + for (int i = 0; i < list.size(); i++) { + array[i] = list.get(i); + } + return array; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java index 27d78f6a..ce1076cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java @@ -11,6 +11,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import org.springframework.stereotype.Service; @@ -22,16 +23,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.Section; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; -import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; @@ -51,13 +51,7 @@ public class DocumentGraphFactory { public DocumentGraph buildDocumentGraph(Document document) { - Context context = new Context(new TableOfContents(), - new LinkedList<>(), - new LinkedList<>(), - new LinkedList<>(), - new LinkedList<>(), - new AtomicInteger(0), - new AtomicInteger(0)); + Context context = new Context(new TableOfContents(), new LinkedList<>(), new LinkedList<>(), new AtomicInteger(0), new AtomicLong(0)); context.pages.addAll(document.getPages().stream().map(this::buildPage).toList()); @@ -79,16 +73,15 @@ public class DocumentGraphFactory { private void addSections(Document document, Context context) { - for (int sectionIdx = 0; sectionIdx < document.getSections().size(); sectionIdx++) { - addSection(document.getSections().get(sectionIdx), sectionIdx, context); + for (var section : document.getSections()) { + addSection(section, context); } } - private void addSection(Section section, int sectionIdx, Context context) { + private void addSection(Section section, Context context) { SectionNode sectionEntity = SectionNode.builder() - .id(sectionIdx) .entities(new LinkedList<>()) .pages(new LinkedList<>()) .paragraphs(new LinkedList<>()) @@ -99,15 +92,19 @@ public class DocumentGraphFactory { context.sections().add(sectionEntity); List pageBlocks = new ArrayList<>(section.getPageBlocks()); + PageNode page = getPage(section.getPageBlocks().get(0).getPage(), context); + sectionEntity.getPages().add(page); + page.getMainBody().add(sectionEntity); if (pageBlocks.get(0) instanceof TextBlock) { sectionEntity.setHeadline(buildAtomicTextBlock(((TextBlock) pageBlocks.get(0)).getSequences(), sectionEntity, context)); + sectionEntity.setNumberOnPage(((TextBlock) pageBlocks.get(0)).getIndexOnPage()); pageBlocks.remove(0); - sectionEntity.getPages().add(getPage(section.getPageBlocks().get(0).getPage(), context)); } else { + sectionEntity.setNumberOnPage(1); sectionEntity.setHeadline(emptyTextBlock(sectionEntity, context)); } - String sectionId = context.tableOfContents.createNewEntryAndReturnId(TableOfContents.SECTION, buildSummary(sectionEntity.getHeadline()), sectionEntity); + String sectionId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, buildSummary(sectionEntity.getHeadline()), sectionEntity); sectionEntity.setTocId(sectionId); int paragraphIdx = 0; @@ -117,7 +114,7 @@ public class DocumentGraphFactory { addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx, context); paragraphIdx++; } else if (abstractTextContainer instanceof Table) { - addTable(sectionEntity, (Table) abstractTextContainer, tableIdx, context); + //addTable(sectionEntity, (Table) abstractTextContainer, tableIdx, context); tableIdx++; } @@ -130,12 +127,11 @@ public class DocumentGraphFactory { PageNode page = getPage(table.getPage(), context); TableNode tableEntity = TableNode.builder().id(tableIdx).tableOfContents(context.tableOfContents()).pages(new LinkedList<>()).parentSection(sectionEntity).build(); sectionEntity.getTables().add(tableEntity); - page.getMainBody().add(tableEntity); if (!page.getMainBody().contains(sectionEntity)) { sectionEntity.getPages().add(page); - page.getMainBody().add(sectionEntity); } + page.getMainBody().add(tableEntity); } @@ -143,19 +139,18 @@ public class DocumentGraphFactory { private void addParagraph(SectionNode sectionEntity, TextBlock originalTextBlock, int paragraphIdx, Context context) { PageNode page = getPage(originalTextBlock.getPage(), context); - ParagraphNode paragraph = ParagraphNode.builder().id(paragraphIdx).numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build(); + ParagraphNode paragraph = ParagraphNode.builder().numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build(); sectionEntity.getParagraphs().add(paragraph); - page.getMainBody().add(paragraph); if (!page.getMainBody().contains(sectionEntity)) { sectionEntity.getPages().add(page); - page.getMainBody().add(sectionEntity); } + page.getMainBody().add(paragraph); var textBlock = buildAtomicTextBlock(originalTextBlock.getSequences(), paragraph, context); paragraph.setAtomicTextBlock(textBlock); - String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), TableOfContents.PARAGRAPH, buildSummary(textBlock), paragraph); + String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), NodeType.PARAGRAPH, buildSummary(textBlock), paragraph); paragraph.setTocId(tocId); } @@ -195,9 +190,7 @@ public class DocumentGraphFactory { private void addFooter(List textBlocks, Context context) { PageNode page = getPage(textBlocks.get(0).getPage(), context); - FooterNode footer = FooterNode.builder().page(page).build(); - AtomicTextBlock atomicTextBlock = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), footer, context); - footer.setAtomicTextBlock(atomicTextBlock); + AtomicTextBlock footer = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), page, context); page.setFooter(footer); } @@ -205,9 +198,7 @@ public class DocumentGraphFactory { public void addHeader(List textBlocks, Context context) { PageNode page = getPage(textBlocks.get(0).getPage(), context); - HeaderNode header = HeaderNode.builder().page(page).build(); - AtomicTextBlock atomicTextBlock = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), header, context); - header.setAtomicTextBlock(atomicTextBlock); + AtomicTextBlock header = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), page, context); page.setHeader(header); } @@ -215,24 +206,18 @@ public class DocumentGraphFactory { private void addEmptyFooter(int pageIndex, Context context) { PageNode page = getPage(pageIndex, context); - FooterNode footer = FooterNode.builder().page(page).build(); - AtomicTextBlock atomicTextBlock = emptyTextBlock(footer, context); - footer.setAtomicTextBlock(atomicTextBlock); - page.setFooter(footer); + page.setFooter(emptyTextBlock(page, context)); } private void addEmptyHeader(int pageIndex, Context context) { PageNode page = getPage(pageIndex, context); - HeaderNode header = HeaderNode.builder().page(page).build(); - AtomicTextBlock atomicTextBlock = emptyTextBlock(header, context); - header.setAtomicTextBlock(atomicTextBlock); - page.setHeader(header); + page.setHeader(emptyTextBlock(page, context)); } - private AtomicTextBlock emptyTextBlock(DocumentNode parent, Context context) { + private AtomicTextBlock emptyTextBlock(DocumentGraphNode parent, Context context) { return AtomicTextBlock.builder() .id(context.textBlockIdx.getAndIncrement()) @@ -252,7 +237,7 @@ public class DocumentGraphFactory { return " probably a table"; } - String[] words = textBlock.getFirstLine().split(" "); + String[] words = textBlock.getFirstLine().toString().split(" "); int bound = Math.min(words.length, 4); List list = new ArrayList<>(Arrays.asList(words).subList(0, bound)); @@ -279,7 +264,7 @@ public class DocumentGraphFactory { } - private AtomicTextBlock buildAtomicTextBlock(List sequences, DocumentNode parent, Context context) { + private AtomicTextBlock buildAtomicTextBlock(List sequences, DocumentGraphNode parent, Context context) { SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences); int offset = context.stringOffset().getAndAdd(searchTextWithTextPositionModel.getSearchText().length()); @@ -312,13 +297,7 @@ public class DocumentGraphFactory { record Context( - TableOfContents tableOfContents, - List pages, - List sections, - List headers, - List footers, - AtomicInteger stringOffset, - AtomicInteger textBlockIdx) { + TableOfContents tableOfContents, List pages, List sections, AtomicInteger stringOffset, AtomicLong textBlockIdx) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java new file mode 100644 index 00000000..65df9534 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java @@ -0,0 +1,171 @@ +package com.iqser.red.service.redaction.v1.server.document.services; +import static java.lang.Math.toIntExact; +import static java.lang.String.format; + +import java.awt.geom.Rectangle2D; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang3.NotImplementedException; +import org.springframework.stereotype.Service; + +import com.google.common.primitives.Ints; +import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; +import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; +import com.iqser.red.service.redaction.v1.server.document.data.PageData; +import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; + +@Service +public class DocumentGraphMapper { + + public DocumentGraph toDocumentGraph(DocumentData documentData) { + + Context context = new Context(documentData, new TableOfContents(), new LinkedList<>(), new LinkedList<>(), documentData.getAtomicTextBlocks()); + + context.pages.addAll(documentData.getPages().stream().map(pageData -> buildPage(pageData, context)).toList()); + buildNodesFromTableOfContents("", context); + DocumentGraph documentGraph= DocumentGraph.builder() + .numberOfPages(documentData.getPages().size()) + .pages(context.pages) + .sections(context.sections) + .tableOfContents(context.tableOfContents) + .build(); + documentGraph.setText(documentGraph.buildTextBlock()); + return documentGraph; + } + + + private void buildNodesFromTableOfContents(String currentTocId, Context context) { + List entries; + if(currentTocId.equals("")) { + entries = context.documentData().getTableOfContents().getEntries(); + } else { + entries = context.documentData().getTableOfContents().get(currentTocId).subEntries(); + } + for (TableOfContentsData.EntryData entryData : entries) { + + switch (entryData.type()) { + case SECTION -> buildSection(entryData, currentTocId, context); + case PARAGRAPH -> buildParagraph(entryData, currentTocId, context); + default -> throw new NotImplementedException("Not yet implemented for type " + entryData.type()); + } + } + } + + + private void buildSection(TableOfContentsData.EntryData entryData, String currentTocId, Context context) { + + SectionNode section = SectionNode.builder() + .entities(new LinkedList<>()) + .pages(new LinkedList<>()) + .paragraphs(new LinkedList<>()) + .tables(new LinkedList<>()) + .subSections(new LinkedList<>()) + .tableOfContents(context.tableOfContents()) + .numberOnPage(entryData.numberOnPage()) + .build(); + + context.sections().add(section); + section.setHeadline(toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(entryData.atomicTextBlock())), section)); + + if (!currentTocId.equals("")) { + SectionNode parent = (SectionNode) context.tableOfContents().getEntryById(currentTocId).node(); + section.setParentSection(parent); + parent.getSubSections().add(section); + } + + PageNode page = getPage(entryData.page(), context); + page.getMainBody().add(section); + section.getPages().add(page); + + String sectionId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, buildSummary(section.getHeadline()), section); + section.setTocId(sectionId); + buildNodesFromTableOfContents(sectionId, context); + } + + + private void buildParagraph(TableOfContentsData.EntryData entryData, String currentTocId, Context context) { + + PageNode page = getPage(entryData.page(), context); + SectionNode parentSection = (SectionNode) context.tableOfContents().getEntryById(currentTocId).node(); + ParagraphNode paragraph = ParagraphNode.builder().numberOnPage(entryData.numberOnPage()).page(page).parentSection(parentSection).build(); + AtomicTextBlock atomicTextBlock = toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(entryData.atomicTextBlock())), paragraph); + paragraph.setAtomicTextBlock(atomicTextBlock); + + if (!page.getMainBody().contains(parentSection)) { + parentSection.getPages().add(page); + } + page.getMainBody().add(paragraph); + + String tocId = context.tableOfContents.createNewChildEntryAndReturnId(currentTocId, NodeType.PARAGRAPH, buildSummary(atomicTextBlock), paragraph); + paragraph.setTocId(tocId); + } + + + private PageNode buildPage(PageData p, Context context) { + + PageNode page = PageNode.builder().height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build(); + AtomicTextBlock header = toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(p.getHeader())), page); + AtomicTextBlock footer = toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(p.getFooter())), page); + page.setHeader(header); + page.setFooter(footer); + return page; + } + + + private static String buildSummary(com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock) { + + if (textBlock == null) { + return " probably a table"; + } + + String[] words = textBlock.getFirstLine().toString().split(" "); + int bound = Math.min(words.length, 4); + List list = new ArrayList<>(Arrays.asList(words).subList(0, bound)); + + return String.join(" ", list); + } + + + private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, DocumentGraphNode parent) { + + return AtomicTextBlock.builder() + .id(atomicTextBlockData.getId()) + .searchText(atomicTextBlockData.getSearchText()) + .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd())) + .lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks())) + .positions(Arrays.stream(atomicTextBlockData.getPositions()) + .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])) + .toList()) + .stringIdxToPositionIdx(Ints.asList(atomicTextBlockData.getStringIdxToPositionIdx())) + .parent(parent) + .build(); + } + + + private PageNode getPage(Long pageIndex, Context context) { + + return context.pages.stream() + .filter(page -> page.getNumber() == toIntExact(pageIndex)) + .findFirst() + .orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex))); + } + + + record Context(DocumentData documentData, TableOfContents tableOfContents, List pages, List sections, List atomicTextBlockData) { + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java index 73ac24f9..de35c4a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java @@ -9,7 +9,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; public class RegexMatcher { - public static boolean anyMatch(String regexPattern, CharSequence searchText) { + public static boolean anyMatch(CharSequence searchText, String regexPattern) { var pattern = Patterns.getCompiledPattern(regexPattern, false); return pattern.matcher(searchText).find(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java index ef784d1c..b0cd5dde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java @@ -93,13 +93,13 @@ public class AbstractTestWithDictionaries { protected final static String TEST_FILE_ID = "123"; @Autowired - private StorageService storageService; + protected StorageService storageService; @MockBean - private DictionaryClient dictionaryClient; + protected DictionaryClient dictionaryClient; @MockBean - private RulesClient rulesClient; + protected RulesClient rulesClient; private final Map> dictionary = new HashMap<>(); private final Map> dossierDictionary = new HashMap<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java index 9e131d65..55506847 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -22,12 +22,14 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.ClassPathResource; import com.iqser.red.service.redaction.v1.model.FileAttribute; +import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.document.services.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; @@ -48,11 +50,16 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { @Autowired private DictionaryService dictionaryService; + @Autowired + private DocumentDataMapper documentDataMapper; + @Qualifier("kieContainer") @Autowired private KieContainer kieContainer; + + @Test @SneakyThrows public void testDroolsOnDocumentGraph() { @@ -227,17 +234,17 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { .forEach(foundEntities::add); }); documentGraph.getPages().forEach( page -> { - TextBlock textBlock = page.getHeader().buildTextBlock(); + TextBlock textBlock = page.getHeader(); searchImplementation.getMatches(textBlock, textBlock.getBoundary().start()) .stream() - .map(bounds -> page.getHeader().createAndAddEntity(bounds, type, entityType)) + .map(bounds -> page.createAndAddEntity(bounds, type, entityType)) .forEach(foundEntities::add); }); documentGraph.getPages().forEach( page -> { - TextBlock textBlock = page.getFooter().buildTextBlock(); + TextBlock textBlock = page.getFooter(); searchImplementation.getMatches(textBlock, textBlock.getBoundary().start()) .stream() - .map(bounds -> page.getFooter().createAndAddEntity(bounds, type, entityType)) + .map(bounds -> page.createAndAddEntity(bounds, type, entityType)) .forEach(foundEntities::add); }); /* diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphMappingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphMappingTest.java new file mode 100644 index 00000000..653af2dc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphMappingTest.java @@ -0,0 +1,55 @@ +package com.iqser.red.service.redaction.v1.server; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; + +import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.services.DocumentDataMapper; +import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory; +import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; + +import lombok.SneakyThrows; + +public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { + + @Autowired + private DocumentGraphFactory documentGraphFactory; + + @Autowired + private PdfSegmentationService segmentationService; + + @Autowired + private DocumentDataMapper documentDataMapper; + + @Autowired + private DocumentGraphMapper documentGraphMapper; + + + @Test + @SneakyThrows + public void testGraphMapping() { + + String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; + + prepareStorage(filename + ".pdf"); + ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); + + var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null); + DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc); + DocumentData documentData = documentDataMapper.toDocumentData(document); + storageService.storeJSONObject(filename + ".json", documentData); + DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData); + + assert document.toString().equals(newDocumentGraph.toString()); + assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString()); + for (int pageIndex = 1; pageIndex < document.getNumberOfPages(); pageIndex++) { + var page = document.getPages().get(pageIndex); + var newPage = document.getPages().get(pageIndex); + assert page.toString().equals(newPage.toString()); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl index 32621953..fd181952 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl @@ -19,8 +19,8 @@ rule "0: Expand CBI Authors with firstname initials" when Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author")) then - section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); - section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); + section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+",dictionary); + section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+",dictionary); end @@ -53,7 +53,7 @@ rule "4: Redact Author(s) cells in Tables with Author(s) header" when Section(hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -61,7 +61,7 @@ rule "5: Redact Author cells in Tables with Author header" when Section(hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -69,7 +69,7 @@ rule "6: Redact and recommand Authors in Tables with Vertebrate study Y/N header when Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")) then - section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -77,8 +77,8 @@ rule "7: Redact if CTL/* or BL/* was found" when Section(searchText.contains("CTL/") || searchText.contains("BL/")) then - section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" ); - section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" ); + section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002", dictionary); + section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002", dictionary); end @@ -86,7 +86,7 @@ rule "8: Redact and add recommendation for et al. author" when Section(searchText.contains("et al")) then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)",dictionary); end @@ -130,7 +130,7 @@ rule "13: Redact Emails by RegEx" when Section(searchText.contains("@")) then - section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -153,25 +153,25 @@ rule "14: Redact contact information" || text.contains("Phone No.") || text.contains("European contact:")) then - section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -179,25 +179,25 @@ rule "15: Redact contact information if applicant is found" when Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")) then - section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -205,17 +205,17 @@ rule "16: Redact contact information if Producer is found" when Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")) then - section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -223,7 +223,7 @@ rule "17: Redact AUTHOR(S)" when Section(searchText.contains("AUTHOR(S):")) then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -231,7 +231,7 @@ rule "18: Redact PERFORMING LABORATORY" when Section(searchText.contains("PERFORMING LABORATORY:")) then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -239,7 +239,7 @@ rule "19: Redact On behalf of Sequani Ltd.:" when Section(searchText.contains("On behalf of Sequani Ltd.: Name Title")) then - section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -247,7 +247,7 @@ rule "20: Redact On behalf of Syngenta Ltd.:" when Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title")) then - section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -257,7 +257,7 @@ rule "21: Purity Hint" when Section(searchText.toLowerCase().contains("purity")) then - section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); + section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl index d178e7f4..78bdf3cf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl @@ -23,7 +23,7 @@ global DocumentGraph document rule "1: Redact CBI_author" when - FileAttribute(label == "Vertebrate Study" && (value.toLowerCase() == "yes")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") entity: EntityNode(type == "CBI_author") then entity.setRedact(true); @@ -33,7 +33,7 @@ rule "1: Redact CBI_author" rule "2: do not redact genitive CBI_author" when - entity: EntityNode(type == "CBI_author", anyMatch("['’’'ʼˈ´`‘′ʻ’']s", textAfter), redact == true) + entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redact == true) then entity.setRedact(false); entity.setEntityType(EntityType.FALSE_POSITIVE); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/headlines.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/headlines.drl index 6a326b69..eff7c311 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/headlines.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/headlines.drl @@ -9,5 +9,5 @@ rule "1: Find headlines" when Section(text.length() > 1) then - section.redactHeadline("headline", 1, "Headline found", "n-a."); + section.redactHeadline("headline", 1, "Headline found", "n-a.",dictionary); end \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_rules.drl new file mode 100644 index 00000000..1a43ea4b --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_rules.drl @@ -0,0 +1,383 @@ +package drools + +import java.util.Set; + +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.redaction.model.* + +global RedactionServiceSettings redactionServiceSettings; +global Dictionary dictionary; + + +// --------------------------------------- AI rules ------------------------------------------------------------------- +rule "-1: find entities from dictionary" + salience 100 + no-loop true + when + section: Section() + then + section.findDictionaryEntities(dictionary, redactionServiceSettings); + update(section); + end + +rule "0: Add CBI_author from ai" + when + section: Section(aiMatchesType("CBI_author")) + then + section.addAiEntities("CBI_author", "CBI_author", dictionary); + end + +rule "0: Combine address parts from ai to CBI_address (org is mandatory)" + when + section: Section(aiMatchesType("ORG")) + then + section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary); + end + +rule "0: Combine address parts from ai to CBI_address (street is mandatory)" + when + section: Section(aiMatchesType("STREET")) + then + section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary); + end + +rule "0: Combine address parts from ai to CBI_address (city is mandatory)" + when + section: Section(aiMatchesType("CITY")) + then + section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary); + end + + +// --------------------------------------- CBI rules ------------------------------------------------------------------- + +rule "1: Redact CBI Authors (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + then + section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "2: Redact CBI Authors (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + then + section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "3: Redact not CBI Address (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + then + section.redactNot("CBI_address", 3, "Address found for non vertebrate study"); + section.ignoreRecommendations("CBI_address"); + end + +rule "4: Redact CBI Address (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + then + section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "5: Do not redact genitive CBI_author" + when + section: Section(matchesType("CBI_author")) + then + section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary); + end + + +rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + end + +rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + end + + +rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + end + +rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + end + + +rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) + then + section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + end + +rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) + then + section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + end + +rule "14: Redact and add recommendation for et al. author (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "et al") + then + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary); + end + +rule "15: Redact and add recommendation for et al. author (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "et al") + then + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary); + end + + +rule "16: Add recommendation for Addresses in Test Organism sections" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText (contains "Species:" && contains "Source:")) + then + section.recommendLineAfter("Source:", "CBI_address"); + end + +rule "17: Add recommendation for Addresses in Test Animals sections" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText (contains "Species" && contains "Source") ) + then + section.recommendLineAfter("Source", "CBI_address"); + end + +/* + +rule "18: Do not redact Names and Addresses if Published Information found" + when + section: Section(matchesType("published_information")) + then + section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found"); + section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found"); + end + +*/ + +// --------------------------------------- PII rules ------------------------------------------------------------------- + + +rule "19: Redacted PII Personal Identification Information (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) + then + section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "20: Redacted PII Personal Identification Information (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) + then + section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "21: Redact Emails by RegEx (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "@") + then + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary); + end + +rule "22: Redact Emails by RegEx (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "@") + then + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + end + + +rule "23: Redact contact information (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study", "Yes") && + (text (contains "Contact point:" || + contains "Contact:" || + contains "Alternative contact:" || + (contains "No:" && contains "Fax") || + (contains "Contact:" && contains "Tel.:") || + contains "European contact:"))) + then + section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + end + +rule "24: Redact contact information (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && + (text (contains "Contact point:" + || contains "Contact:" + || contains"Alternative contact:" + || (contains "No:" && contains "Fax") + || (contains "Contact:" && contains "Tel.:") + || contains "European contact:" + ))) + then + section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + end + +rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( + text contains "Contact" + || contains "Telephone" + || contains "Phone" + || contains "Fax" + || contains "Tel" + || contains "Ter" + || contains "Mobile" + || contains "Fel" + || contains "Fer" + )) + then + section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary); + end + +rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( + text contains "Contact" + || contains "Telephone" + || contains "Phone" + || contains "Fax" + || contains "Tel" + || contains "Ter" + || contains "Mobile" + || contains "Fel" + || contains "Fer" + )) + then + section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary); + end + +rule "27: Redact AUTHOR(S) (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && (searchText contains "AUTHOR(S):" + && contains "COMPLETION DATE:" + && not contains "STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary); + end + +rule "28: Redact AUTHOR(S) (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && (searchText contains "AUTHOR(S):" + && contains "COMPLETION DATE:" + && not contains "STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary); + end + + +rule "29: Redact AUTHOR(S) (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && (searchText contains "AUTHOR(S):" + && contains "STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary); + end + +rule "30: Redact AUTHOR(S) (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText contains "AUTHOR(S):" + && contains "STUDY COMPLETION DATE:" + ) + then + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary); + end + + +rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText contains "PERFORMING LABORATORY:" + ) + then + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary); + section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); + end + +rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText contains "PERFORMING LABORATORY:") + then + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary); + end + +rule "33: Redact study director abbreviation" + when + section: Section(searchText contains "KATH" || contains "BECH" || contains "KML") + then + section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary); + end + + +// --------------------------------------- other rules ------------------------------------------------------------------- + +rule "34: Purity Hint" + when + section: Section(searchText.toLowerCase() contains "purity") + then + section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary); + end + + + +rule "35: Redact signatures (Non vertebrate study)" + when + section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) + then + section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "36: Redact signatures (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) + then + section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "43: Redact Logos (Vertebrate study)" + when + section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo")) + then + section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 8695d99f..9381f971 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1,67 +1,40 @@ package drools import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch; import java.util.List; import java.util.LinkedList; import java.util.HashSet; -import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.server.document.graph.* +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.* +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import java.util.Set; +global DocumentGraph document -global Dictionary dictionary -query "getEntities" - $result: Entity() -end -// --------------------------------------- AI rules ------------------------------------------------------------------- -rule "0: find entities from entry dictionary" - salience 100 +rule "1: Redact CBI_author" + when - $section: Section() - $paragraph: Paragraph(sectionNumber == $section.sectionNumber) - $entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition()) + FileAttribute(label == "Vertebrate Study" && (value.toLowerCase() == "yes")) + entity: EntityNode(type == "CBI_author") then - List eps = new LinkedList<>(); - eps.add($entityPositionSequence); - Set engineSet = new HashSet<>(); - engineSet.add(Engine.DICTIONARY); - Set empty = new HashSet<>(); - insert(new Entity("CBI_author", - "custom", - true, - format("Found in Dictionary: %s", "custom"), - eps, - $section.getHeadline(), - 0, - $section.getSectionNumber(), - $paragraph.getParagraphNumber(), - "", - true, - "", - "", - -1, - -1, - false, - engineSet, - empty, - EntityType.ENTITY - )); + entity.setRedact(true); + update(entity) end -/* -rule "44: Don't redact Author if contained in paragraph with published information" - when - $section: Section() - $paragraph: Paragraph($section.sectionNumber == sectionNumber) +rule "2: do not redact genitive CBI_author" - $authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber()) - $publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber()) + when + entity: EntityNode(type == "CBI_author", anyMatch("['’’'ʼˈ´`‘′ʻ’']s", textAfter)) then - $authorEntity.setRedaction(false); + entity.setRedact(false); + entity.setEntityType(EntityType.FALSE_POSITIVE); + update(entity) end -*/ \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl new file mode 100644 index 00000000..3c498213 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl @@ -0,0 +1 @@ +rules.drl \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl index ad7726cd..97c7ac3f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl @@ -11,28 +11,28 @@ rule "0: Add CBI_author from ai" when Section(aiMatchesType("CBI_author")) then - section.addAiEntities("CBI_author", "CBI_author"); + section.addAiEntities("CBI_author", "CBI_author",dictionary); end rule "0: Combine address parts from ai to CBI_address (org is mandatory)" when Section(aiMatchesType("ORG")) then - section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); + section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary); end rule "0: Combine address parts from ai to CBI_address (street is mandatory)" when Section(aiMatchesType("STREET")) then - section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); + section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary); end rule "0: Combine address parts from ai to CBI_address (city is mandatory)" when Section(aiMatchesType("CITY")) then - section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false); + section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary); end @@ -73,7 +73,7 @@ rule "5: Do not redact genitive CBI_author" when Section(matchesType("CBI_author")) then - section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); + section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary); end @@ -81,14 +81,14 @@ rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -96,14 +96,14 @@ rule "8: Redact Author cells in Tables with Author header (Non vertebrate study) when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -111,14 +111,14 @@ rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N heade when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) then - section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) then - section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end /* Syngenta specific laboratory rule */ @@ -133,14 +133,14 @@ rule "14: Redact and add recommendation for et al. author (Non vertebrate study) when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end rule "15: Redact and add recommendation for et al. author (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -190,14 +190,14 @@ rule "21: Redact Emails by RegEx (Non vertebrate study)" when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end rule "22: Redact Emails by RegEx (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -224,25 +224,25 @@ rule "23: Redact contact information (Non vertebrate study)" || text.contains("European contact:") )) then - section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end rule "24: Redact contact information (Vertebrate study)" @@ -268,25 +268,25 @@ rule "24: Redact contact information (Vertebrate study)" || text.contains("European contact:") )) then - section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -305,7 +305,7 @@ rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)" || text.contains("Fer") )) then - section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" @@ -323,7 +323,7 @@ rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" || text.contains("Fer") )) then - section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -335,7 +335,7 @@ rule "27: Redact AUTHOR(S) (Non vertebrate study)" && !searchText.contains("STUDY COMPLETION DATE:") ) then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end rule "28: Redact AUTHOR(S) (Vertebrate study)" @@ -346,7 +346,7 @@ rule "28: Redact AUTHOR(S) (Vertebrate study)" && !searchText.contains("STUDY COMPLETION DATE:") ) then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -357,7 +357,7 @@ rule "29: Redact AUTHOR(S) (Non vertebrate study)" && searchText.contains("STUDY COMPLETION DATE:") ) then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end rule "30: Redact AUTHOR(S) (Vertebrate study)" @@ -367,7 +367,7 @@ rule "30: Redact AUTHOR(S) (Vertebrate study)" && searchText.contains("STUDY COMPLETION DATE:") ) then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -377,7 +377,7 @@ rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" && searchText.contains("PERFORMING LABORATORY:") ) then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); end @@ -386,7 +386,7 @@ rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:")) then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -396,7 +396,7 @@ rule "33: Purity Hint" when Section(searchText.toLowerCase().contains("purity")) then - section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); + section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 939af4b9..526a95a3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -14,15 +14,21 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import java.util.Set; -global DocumentGraph document; +global DocumentGraph document +rule "dummy" + when + then + System.out.println("This actually works"); + end rule "1: Redact CBI_author" when - FileAttribute(label == "Vertebrate Study" && (value == "Yes" || value == "Y" || value == "y")) - $entity: Entity2(type == "CBI_author") + //FileAttribute(label == "Vertebrate Study" && (value == "Yes")) + $entity: EntityNode(type == "CBI_author") then + System.out.println("Entity found"); modify($entity){ $entity.setRedact(true) } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index 65841442..03c18c70 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -10,7 +10,7 @@ rule "0: Add CBI_author from ai" when Section(aiMatchesType("CBI_author")) then - section.addAiEntities("CBI_author", "CBI_author"); + section.addAiEntities("CBI_author", "CBI_author",dictionary); end @@ -18,7 +18,7 @@ rule "0: Combine ai types CBI_author from ai" when Section(aiMatchesType("ORG")) then - section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); + section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary); end @@ -59,7 +59,7 @@ rule "5: Do not redact genitive CBI_author" when Section(matchesType("CBI_author")) then - section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); + section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary); end @@ -67,7 +67,7 @@ rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -75,7 +75,7 @@ rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate stud when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -83,7 +83,7 @@ rule "8: Redact Author cells in Tables with Author header (Non vertebrate study) when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -91,7 +91,7 @@ rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) then - section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -99,7 +99,7 @@ rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N heade when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) then - section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -107,7 +107,7 @@ rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N heade when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) then - section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -115,7 +115,7 @@ rule "13: Redact addresses that start with BL or CTL" when Section(searchText.contains("BL") || searchText.contains("CT")) then - section.redactNotAndRecommendByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address", 13, "Laboratory for vertebrate studies found"); + section.redactNotAndRecommendByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address", 13, "Laboratory for vertebrate studies found",dictionary); end @@ -123,7 +123,7 @@ rule "14: Redact and add recommendation for et al. author (Non vertebrate study) when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -131,7 +131,7 @@ rule "15: Redact and add recommendation for et al. author (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -183,7 +183,7 @@ rule "21: Redact Emails by RegEx (Non vertebrate study)" when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -191,7 +191,7 @@ rule "22: Redact Emails by RegEx (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -199,7 +199,7 @@ rule "23: Redact telephone numbers by RegEx (Non vertebrate study)" when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true)) then - section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 23, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 23, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -207,7 +207,7 @@ rule "24: Redact telephone numbers by RegEx (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true)) then - section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 24, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 24, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -238,25 +238,25 @@ rule "26: Redact contact information (Non vertebrate study)" || text.contains("Phone No.") || text.contains("European contact:"))) then - section.redactLineAfter("Contact point:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Email:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("e-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail address:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone No:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact point:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Email:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("e-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail address:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone No:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -279,25 +279,25 @@ rule "27: Redact contact information (Vertebrate study)" || text.contains("Phone No.") || text.contains("European contact:"))) then - section.redactLineAfter("Contact point:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Email:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("e-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail address:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone No:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact point:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Email:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("e-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail address:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone No:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -305,25 +305,25 @@ rule "28: Redact contact information if applicant is found (Non vertebrate study when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))) then - section.redactLineAfter("Contact point:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Email:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("e-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail address:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone No:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact point:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Email:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("e-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail address:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone No:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -331,25 +331,25 @@ rule "29: Redact contact information if applicant is found (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))) then - section.redactLineAfter("Contact point:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Email:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("e-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail address:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone No:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact point:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Email:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("e-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail address:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Alternative contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone No:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("Contact:", "Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("European contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -357,17 +357,17 @@ rule "30: Redact contact information if Producer is found (Non vertebrate study) when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))) then - section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -375,17 +375,17 @@ rule "31: Redact contact information if Producer is found (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))) then - section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("E-mail:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Fax number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Telephone number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Tel:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Phone No.", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("E-mail:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Fax number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Telephone number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Tel:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactLineAfter("Phone No.", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + section.redactBetween("No:", "Fax", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -393,7 +393,7 @@ rule "32: Redact AUTHOR(S) (Non vertebrate study)" when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:")) then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 32, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 32, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -401,7 +401,7 @@ rule "33: Redact AUTHOR(S) (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:")) then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 33, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 33, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -409,7 +409,7 @@ rule "34: Redact AUTHOR(S) (Non vertebrate study)" when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:")) then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 34, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 34, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); end @@ -417,7 +417,7 @@ rule "35: Redact AUTHOR(S) (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:")) then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 35, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 35, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -425,7 +425,7 @@ rule "36: Redact PERFORMING LABORATORY (Non vertebrate study)" when Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:")) then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 36, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 36, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); section.redactNot("CBI_address", 36, "Performing laboratory found for non vertebrate study"); end @@ -434,7 +434,7 @@ rule "37: Redact PERFORMING LABORATORY (Vertebrate study)" when Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:")) then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 37, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 37, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); end @@ -444,7 +444,7 @@ rule "50: Purity Hint" when Section(searchText.toLowerCase().contains("purity")) then - section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); + section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary); end