Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6093: Prototype document structure

Browse Source 
*refactored File Structure
*started refactor of original rules
This commit is contained in:
Kilian Schuettler 2023-03-07 14:20:27 +01:00
parent 7879bcd909
commit 741290a687
36 changed files with 1329 additions and 609 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java Normal file
View File

@ -0,0 +1,21 @@
package com.iqser.red.service.redaction.v1.server.document.data;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class AtomicTextBlockData {
Long id;
String searchText;
int start;
int end;
int[] lineBreaks;
int[] stringIdxToPositionIdx;
float[][] positions;
}

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java Normal file
View File

@ -0,0 +1,19 @@
package com.iqser.red.service.redaction.v1.server.document.data;
import java.util.List;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DocumentData {
List<PageData> pages;
List<AtomicTextBlockData> atomicTextBlocks;
TableOfContentsData tableOfContents;
}

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java Normal file
View File

@ -0,0 +1,20 @@
package com.iqser.red.service.redaction.v1.server.document.data;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class PageData {
int number;
int height;
int width;
Long header;
Long footer;
}

52
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java Normal file
View File

@ -0,0 +1,52 @@
package com.iqser.red.service.redaction.v1.server.document.data;
import static java.lang.String.format;
import java.util.Arrays;
import java.util.List;
import javax.management.openmbean.InvalidKeyException;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class TableOfContentsData {
List<EntryData> entries;
public EntryData get(String tocId) {
List<Integer> ids = getIds(tocId);
if (ids.size() < 1) {
throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", tocId));
}
EntryData entry = entries.get(ids.get(0));
for (int id : ids.subList(1, ids.size())) {
entry = entry.subEntries().get(id);
}
return entry;
}
private static List<Integer> getIds(String idsAsString) {
return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList();
}
@Builder
public record EntryData(String tocId, List<EntryData> subEntries, NodeType type, Long atomicTextBlock, Long page, int numberOnPage) {
}
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java
View File

@ -18,7 +18,9 @@ public class Boundary {
this.end = end; this.end = end;
} }
public int length() { public int length() {
return end - start; return end - start;
} }
@ -76,9 +78,11 @@ public class Boundary {
return contains(boundary.start()) || contains(boundary.end()); return contains(boundary.start()) || contains(boundary.end());
} }
@Override @Override
public String toString() { public String toString() {
return format("Range [%d|%d)", start, end);
return format("Boundary [%d|%d)", start, end);
} }
} }

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java
View File

@ -7,10 +7,11 @@ import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
@ -40,7 +41,17 @@ public class DocumentGraph {
public ConcatenatedTextBlock buildTextBlock() { public ConcatenatedTextBlock buildTextBlock() {
return streamAllNodes().filter(DocumentNode::isTerminal).map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); return streamAtomicTextBlocksInOrder().collect(new TextBlockCollector());
}
public Stream<AtomicTextBlock> streamAtomicTextBlocksInOrder() {
return Stream.concat(//
streamAllNodes().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getAtomicTextBlock),//
Stream.concat(//
pages.stream().map(PageNode::getHeader),//
pages.stream().map(PageNode::getFooter)));
} }
@ -64,20 +75,15 @@ public class DocumentGraph {
} }
public Set<EntityNode> getEntities() { public Set<EntityNode> getEntities() {
return streamAllNodes().filter(DocumentNode::isTerminal).map(DocumentNode::getEntities).flatMap(List::stream).collect(Collectors.toSet()); return streamAllNodes().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getEntities).flatMap(List::stream).collect(Collectors.toSet());
} }
private Stream<DocumentNode> streamAllNodes() { private Stream<DocumentGraphNode> streamAllNodes() {
return Stream.concat(// return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node);
tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node), //
Stream.concat(//
pages.stream().map(PageNode::getHeader), //
pages.stream().map(PageNode::getFooter))); //
} }

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java
View File

@ -11,18 +11,14 @@ import java.util.stream.Stream;
import javax.management.openmbean.InvalidKeyException; import javax.management.openmbean.InvalidKeyException;
import com.google.common.hash.Hashing; import com.google.common.hash.Hashing;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import lombok.Data; import lombok.Data;
@Data @Data
public class TableOfContents { public class TableOfContents {
public static final String SECTION = "Sec";
public static final String HEADLINE = "H";
public static final String PARAGRAPH = "Par";
public static final String TABLE = "Tab";
List<Entry> entries; List<Entry> entries;
@ -32,19 +28,19 @@ public class TableOfContents {
} }
public String createNewEntryAndReturnId(String keyword, String summary, DocumentNode node) { public String createNewEntryAndReturnId(NodeType nodeType, String summary, DocumentGraphNode node) {
String id = String.format("%d", entries.size()); String id = String.format("%d", entries.size());
entries.add(new Entry(keyword, id, summary, new LinkedList<>(), node)); entries.add(new Entry(nodeType, id, summary, new LinkedList<>(), node));
return id; return id;
} }
public String createNewChildEntryAndReturnId(String parentId, String keyword, String summary, DocumentNode node) { public String createNewChildEntryAndReturnId(String parentId, NodeType nodeType, String summary, DocumentGraphNode node) {
Entry parent = getEntryById(parentId); Entry parent = getEntryById(parentId);
String childId = parentId + String.format(".%d", parent.children().size()); String childId = parentId + String.format(".%d", parent.children().size());
parent.children().add(new Entry(keyword, childId, summary, new LinkedList<>(), node)); parent.children().add(new Entry(nodeType, childId, summary, new LinkedList<>(), node));
return childId; return childId;
} }
@ -100,7 +96,7 @@ public class TableOfContents {
} }
public record Entry(String type, String id, String summary, List<Entry> children, DocumentNode node) { public record Entry(NodeType type, String id, String summary, List<Entry> children, DocumentGraphNode node) {
@Override @Override
public String toString() { public String toString() {
@ -108,6 +104,7 @@ public class TableOfContents {
return id + ": " + type + ".: " + summary; return id + ": " + type + ".: " + summary;
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Hashing.murmur3_32_fixed().hashString(type + id + summary + children.hashCode(), StandardCharsets.UTF_8).hashCode(); return Hashing.murmur3_32_fixed().hashString(type + id + summary + children.hashCode(), StandardCharsets.UTF_8).hashCode();

43
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentNode.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentGraphNode.java
View File

@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentUtility.enrichEntity; import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentUtility.enrichEntity;
import static java.lang.String.format; import static java.lang.String.format;
import java.util.List; import java.util.Set;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
@ -12,7 +12,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBl
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
public interface DocumentNode { public interface DocumentGraphNode {
/** /**
* Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity. * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity.
@ -24,12 +24,12 @@ public interface DocumentNode {
/** /**
* Any Node maintains its own List of Entities. * Any Node maintains its own Set of Entities.
* This List contains all Entities, whose first index is located in any of the AtomicTextBlocks underneath this Node. * This Set contains all Entities, whose first index is located in any of the AtomicTextBlocks underneath this Node.
* *
* @return List of all Entities associated with this Node * @return Set of all Entities associated with this Node
*/ */
List<EntityNode> getEntities(); Set<EntityNode> getEntities();
/** /**
@ -49,10 +49,27 @@ public interface DocumentNode {
* *
* @return Node that represents the Parent or null, if no parent is present. * @return Node that represents the Parent or null, if no parent is present.
*/ */
DocumentNode getParent(); DocumentGraphNode getParent();
Stream<DocumentNode> streamAllSubNodes(); Stream<DocumentGraphNode> streamAllSubNodes();
/**
* Each AtomicTextBlock has a number assigned per page, this returns the number of the first AtomicTextBlock underneath this node
*
* @return Integer representing the number on the page
*/
Integer getNumberOnPage();
/**
*
* @return the fist headline whent traversing the tree upwards
*/
default CharSequence getHeadline() {
return getParent().getHeadline();
}
/** /**
@ -165,7 +182,7 @@ public interface DocumentNode {
private void setFields(EntityNode entity, AtomicTextBlock atomicTextBlock) { private void setFields(EntityNode entity, AtomicTextBlock atomicTextBlock) {
if (atomicTextBlock.containsRange(entity.getBoundary())) { if (atomicTextBlock.containsBoundary(entity.getBoundary())) {
enrichEntity(entity, atomicTextBlock); enrichEntity(entity, atomicTextBlock);
} else { } else {
this.setFieldsFromParents(this, entity); this.setFieldsFromParents(this, entity);
@ -175,7 +192,7 @@ public interface DocumentNode {
private void addEntityToParents(EntityNode entity) { private void addEntityToParents(EntityNode entity) {
DocumentNode node = this; DocumentGraphNode node = this;
while (node.hasParent()) { while (node.hasParent()) {
node = node.getParent(); node = node.getParent();
node.getEntities().add(entity); node.getEntities().add(entity);
@ -184,12 +201,12 @@ public interface DocumentNode {
} }
private void setFieldsFromParents(DocumentNode node, EntityNode entity) { private void setFieldsFromParents(DocumentGraphNode node, EntityNode entity) {
if (node.hasParent()) { if (node.hasParent()) {
DocumentNode parent = node.getParent(); DocumentGraphNode parent = node.getParent();
TextBlock textBlock = parent.buildTextBlock(); TextBlock textBlock = parent.buildTextBlock();
if (textBlock.containsRange(entity.getBoundary())) { if (textBlock.containsBoundary(entity.getBoundary())) {
enrichEntity(entity, textBlock); enrichEntity(entity, textBlock);
return; return;
} else { } else {

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java
View File

@ -7,7 +7,9 @@ import java.util.List;
import java.util.Set; import java.util.Set;
import com.google.common.hash.Hashing; import com.google.common.hash.Hashing;
import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import lombok.AccessLevel; import lombok.AccessLevel;
@ -34,24 +36,41 @@ public class EntityNode {
EntityType entityType; EntityType entityType;
@Builder.Default @Builder.Default
boolean redact = false; boolean redaction = false;
@Builder.Default @Builder.Default
boolean recommend = false; boolean falsePositive = false;
@Builder.Default @Builder.Default
boolean removed = false; boolean removed = false;
@Builder.Default
boolean ignored = false;
@Builder.Default
boolean resized = false;
@Builder.Default
boolean skipRemoveEntitiesContainedInLarger = false;
@Builder.Default
boolean isDictionaryEntry = false;
@Builder.Default
Set<Engine> engines = new HashSet<>();
@Builder.Default
Set<Entity> references = new HashSet<>();
@Builder.Default
int matchedRule = -1;
@Builder.Default
String redactionReason = "";
@Builder.Default
String legalBasis = "";
// inferrable from graph // inferrable from graph
int uniqueId;
String value; String value;
CharSequence textBefore; CharSequence textBefore;
CharSequence textAfter; CharSequence textAfter;
PageNode page; PageNode page;
List<Rectangle2D> positions; List<Rectangle2D> positions;
@Builder.Default @Builder.Default
Set<DocumentNode> containingNodes = new HashSet<>(); Set<DocumentGraphNode> containingNodes = new HashSet<>();
public void addContainingNode(DocumentNode containingNode) { public void addContainingNode(DocumentGraphNode containingNode) {
containingNodes.add(containingNode); containingNodes.add(containingNode);
} }

71
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java
View File

@ -1,71 +0,0 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class FooterNode implements DocumentNode {
@EqualsAndHashCode.Exclude
PageNode page;
AtomicTextBlock atomicTextBlock;
@Builder.Default
@EqualsAndHashCode.Exclude
List<EntityNode> entities = new LinkedList<>();
@Override
public AtomicTextBlock buildTextBlock() {
return atomicTextBlock;
}
@Override
public DocumentNode getParent() {
throw new UnsupportedOperationException("Footer has no Parent, use getPage() instead");
}
@Override
public Stream<DocumentNode> streamAllSubNodes() {
return Stream.of(this);
}
@Override
public boolean hasParent() {
return false;
}
@Override
public boolean isTerminal() {
return true;
}
@Override
public String toString() {
return atomicTextBlock.toString();
}
}

72
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java
View File

@ -1,72 +0,0 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class HeaderNode implements DocumentNode {
@EqualsAndHashCode.Exclude
PageNode page;
AtomicTextBlock atomicTextBlock;
@Builder.Default
@EqualsAndHashCode.Exclude
List<EntityNode> entities = new LinkedList<>();
@Override
public AtomicTextBlock buildTextBlock() {
return atomicTextBlock;
}
@Override
public DocumentNode getParent() {
throw new UnsupportedOperationException("Header has no Parent, use getPage() instead");
}
@Override
public Stream<DocumentNode> streamAllSubNodes() {
return Stream.of(this);
}
@Override
public boolean hasParent() {
return false;
}
@Override
public boolean isTerminal() {
return true;
}
@Override
public String toString() {
return atomicTextBlock.toString();
}
}

28
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java Normal file
View File

@ -0,0 +1,28 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
public enum NodeType {
SECTION {
public String toString() {
return "Section";
}
},
PARAGRAPH {
public String toString() {
return "Paragraph";
}
},
TABLE {
public String toString() {
return "Table";
}
},
TABLE_CELL {
public String toString() {
return "Cell";
}
}
}

61
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java
View File

@ -1,11 +1,13 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import java.util.LinkedList; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -18,42 +20,24 @@ import lombok.experimental.FieldDefaults;
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class PageNode implements DocumentNode { public class PageNode implements DocumentGraphNode{
Integer number; Integer number;
Integer height; Integer height;
Integer width; Integer width;
List<DocumentNode> mainBody; List<DocumentGraphNode> mainBody;
HeaderNode header; AtomicTextBlock header;
FooterNode footer; AtomicTextBlock footer;
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
List<EntityNode> entities = new LinkedList<>(); Set<EntityNode> entities = new HashSet<>();
@Override
public ConcatenatedTextBlock buildTextBlock() { public ConcatenatedTextBlock buildTextBlock() {
return mainBody.stream().filter(DocumentNode::isTerminal).map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); return mainBody.stream().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getAtomicTextBlock).collect(new TextBlockCollector());
}
@Override
public DocumentNode getParent() {
throw new UnsupportedOperationException("A Page has no parent!");
}
@Override
public Stream<DocumentNode> streamAllSubNodes() {
return Stream.concat(//
mainBody.stream(), //
Stream.concat(//
Stream.of(header), //
Stream.of(footer))); //
} }
@ -64,6 +48,13 @@ public class PageNode implements DocumentNode {
} }
@Override
public DocumentGraphNode getParent() {
return null;
}
@Override @Override
public boolean hasParent() { public boolean hasParent() {
@ -71,10 +62,24 @@ public class PageNode implements DocumentNode {
} }
@Override
public Stream<DocumentGraphNode> streamAllSubNodes() {
return mainBody.stream();
}
@Override
public Integer getNumberOnPage() {
return 0;
}
@Override @Override
public String toString() { public String toString() {
return number.toString(); return header.getSearchText() + buildTextBlock().toString() + footer.getSearchText();
} }
} }

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java
View File

@ -1,7 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import java.util.LinkedList; import java.util.HashSet;
import java.util.List; import java.util.Set;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
@ -17,10 +17,9 @@ import lombok.experimental.FieldDefaults;
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class ParagraphNode implements DocumentNode { public class ParagraphNode implements DocumentGraphNode {
String tocId; String tocId;
Integer id;
Integer numberOnPage; Integer numberOnPage;
Integer numberInSection; Integer numberInSection;
@ -32,7 +31,7 @@ public class ParagraphNode implements DocumentNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
List<EntityNode> entities = new LinkedList<>(); Set<EntityNode> entities = new HashSet<>();
@Override @Override
@ -43,7 +42,7 @@ public class ParagraphNode implements DocumentNode {
@Override @Override
public DocumentNode getParent() { public DocumentGraphNode getParent() {
return parentSection; return parentSection;
} }
@ -63,8 +62,9 @@ public class ParagraphNode implements DocumentNode {
} }
@Override @Override
public Stream<DocumentNode> streamAllSubNodes() { public Stream<DocumentGraphNode> streamAllSubNodes() {
return Stream.of(this); return Stream.of(this);
} }
} }

17
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java
View File

@ -1,7 +1,8 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import java.util.LinkedList; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
@ -22,14 +23,14 @@ import lombok.extern.slf4j.Slf4j;
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class SectionNode implements DocumentNode { public class SectionNode implements DocumentGraphNode {
String tocId; String tocId;
Integer id; Integer numberOnPage;
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
TableOfContents tableOfContents; TableOfContents tableOfContents;
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
DocumentNode parentSection; DocumentGraphNode parentSection;
AtomicTextBlock headline; AtomicTextBlock headline;
@ -42,7 +43,7 @@ public class SectionNode implements DocumentNode {
@Builder.Default @Builder.Default
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
List<EntityNode> entities = new LinkedList<>(); Set<EntityNode> entities = new HashSet<>();
@Override @Override
@ -55,12 +56,12 @@ public class SectionNode implements DocumentNode {
@Override @Override
public ConcatenatedTextBlock buildTextBlock() { public ConcatenatedTextBlock buildTextBlock() {
return streamAllSubNodes().map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); return streamAllSubNodes().map(DocumentGraphNode::getAtomicTextBlock).collect(new TextBlockCollector());
} }
@Override @Override
public DocumentNode getParent() { public DocumentGraphNode getParent() {
if (hasParent()) { if (hasParent()) {
return parentSection; return parentSection;
@ -71,7 +72,7 @@ public class SectionNode implements DocumentNode {
@Override @Override
public Stream<DocumentNode> streamAllSubNodes() { public Stream<DocumentGraphNode> streamAllSubNodes() {
return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node); return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node);
} }

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java
View File

@ -17,10 +17,11 @@ import lombok.experimental.FieldDefaults;
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class TableCellNode implements DocumentNode { public class TableCellNode implements DocumentGraphNode {
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
TableNode parentTable; TableNode parentTable;
Integer numberOnPage;
AtomicTextBlock atomicTextBlock; AtomicTextBlock atomicTextBlock;
PageNode page; PageNode page;
@ -37,7 +38,7 @@ public class TableCellNode implements DocumentNode {
@Override @Override
public DocumentNode getParent() { public DocumentGraphNode getParent() {
return parentTable; return parentTable;
} }
@ -50,8 +51,9 @@ public class TableCellNode implements DocumentNode {
} }
@Override @Override
public Stream<DocumentNode> streamAllSubNodes() { public Stream<DocumentGraphNode> streamAllSubNodes() {
return Stream.of(this); return Stream.of(this);
} }
} }

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java
View File

@ -20,12 +20,13 @@ import lombok.experimental.FieldDefaults;
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class TableNode implements DocumentNode { public class TableNode implements DocumentGraphNode {
Integer id; Integer id;
String tocId; String tocId;
Integer numberOfRows; Integer numberOfRows;
Integer numberOfCols; Integer numberOfCols;
Integer numberOnPage;
List<TableCellNode> tableHeaders; List<TableCellNode> tableHeaders;
List<List<TableCellNode>> tableCells; List<List<TableCellNode>> tableCells;
TableOfContents tableOfContents; TableOfContents tableOfContents;
@ -65,14 +66,14 @@ public class TableNode implements DocumentNode {
} }
@Override @Override
public Stream<DocumentNode> streamAllSubNodes() { public Stream<DocumentGraphNode> streamAllSubNodes() {
return streamTableCells().map(Function.identity()); return streamTableCells().map(Function.identity());
} }
@Override @Override
public DocumentNode getParent() { public DocumentGraphNode getParent() {
return parentSection; return parentSection;
} }

30
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java
View File

@ -6,7 +6,7 @@ import java.awt.geom.Rectangle2D;
import java.util.List; import java.util.List;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -21,7 +21,7 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class AtomicTextBlock implements TextBlock { public class AtomicTextBlock implements TextBlock {
Integer id; Long id;
//string coordinates //string coordinates
Boundary boundary; Boundary boundary;
@ -33,7 +33,7 @@ public class AtomicTextBlock implements TextBlock {
List<Rectangle2D> positions; List<Rectangle2D> positions;
@EqualsAndHashCode.Exclude @EqualsAndHashCode.Exclude
DocumentNode parent; DocumentGraphNode parent;
public int indexOf(String searchTerm) { public int indexOf(String searchTerm) {
@ -80,21 +80,19 @@ public class AtomicTextBlock implements TextBlock {
} }
public List<Rectangle2D> getPositions(Boundary range) { public List<Rectangle2D> getPositions(Boundary boundary) {
if (!containsRange(range)) { if (!containsBoundary(boundary)) {
throw new IndexOutOfBoundsException(format("String index Range [%d|%d) is out of boundary for String index [%d|%d)", throw new IndexOutOfBoundsException(format("%s is out of bounds for %s",
range.start(), boundary,
range.end(), this.boundary));
range.start(),
range.end()));
} }
if (range.end() == this.boundary.end()) { if (boundary.end() == this.boundary.end()) {
return positions.subList(stringIdxToPositionIdx.get(range.start() - this.boundary.start()), positions.size()); return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), positions.size());
} }
return positions.subList(stringIdxToPositionIdx.get(range.start() - this.boundary.start()), stringIdxToPositionIdx.get(range.end() - this.boundary.start())); return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), stringIdxToPositionIdx.get(boundary.end() - this.boundary.start()));
} }
@ -119,12 +117,6 @@ public class AtomicTextBlock implements TextBlock {
} }
public String getFirstLine() {
return searchText.substring(0, getNextLinebreak(0) - boundary.start());
}
@Override @Override
public String toString() { public String toString() {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java
View File

@ -51,7 +51,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
boundary.setStart(textBlock.getBoundary().start()); boundary.setStart(textBlock.getBoundary().start());
boundary.setEnd(textBlock.getBoundary().end()); boundary.setEnd(textBlock.getBoundary().end());
} else if (boundary.end() != textBlock.getBoundary().start()) { } else if (boundary.end() != textBlock.getBoundary().start()) {
throw new UnsupportedOperationException(format("Can only concat consecutive TextBlocks, trying to concat %s and %s", textBlock.getBoundary(), boundary)); throw new UnsupportedOperationException(format("Can only concat consecutive TextBlocks, trying to concat %s to %s", textBlock.getBoundary(), boundary));
} }
this.searchText.append(textBlock.getSearchText()); this.searchText.append(textBlock.getSearchText());
this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks()); this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks());

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java
View File

@ -36,12 +36,18 @@ public interface TextBlock extends CharSequence {
int indexOf(String searchTerm); int indexOf(String searchTerm);
default boolean containsRange(Boundary range) { default CharSequence getFirstLine() {
if (range.end() < range.start()) { return subSequence(getBoundary().start(), getNextLinebreak(getBoundary().start()));
throw new IllegalArgumentException(format("Invalid range [%d|%d), StartIndex must be smaller than EndIndex", range.start(), range.end()));
} }
return getBoundary().contains(range);
default boolean containsBoundary(Boundary boundary) {
if (boundary.end() < boundary.start()) {
throw new IllegalArgumentException(format("Invalid %s, StartIndex must be smaller than EndIndex", boundary));
}
return getBoundary().contains(boundary);
} }

98
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentDataMapper.java Normal file
View File

@ -0,0 +1,98 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import java.awt.geom.Rectangle2D;
import java.util.List;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
@Service
public class DocumentDataMapper {
public DocumentData toDocumentData(DocumentGraph documentGraph) {
List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamAtomicTextBlocksInOrder().map(this::toAtomicTextBlockData).toList();
List<PageData> pageData = documentGraph.getPages().stream().map(this::toPageData).toList();
TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
return DocumentData.builder().atomicTextBlocks(atomicTextBlockData).pages(pageData).tableOfContents(tableOfContentsData).build();
}
private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) {
return new TableOfContentsData(tableOfContents.getEntries().stream().map(this::toEntryData).toList());
}
private TableOfContentsData.EntryData toEntryData(TableOfContents.Entry entry) {
return TableOfContentsData.EntryData.builder()
.tocId(entry.id())
.subEntries(entry.children().stream().map(this::toEntryData).toList())
.type(entry.type())
.atomicTextBlock(entry.node().isTerminal() ? entry.node().getAtomicTextBlock().getId() : -1L)
.page(Long.valueOf(entry.node().getPage().getNumber()))
.numberOnPage(entry.node().getNumberOnPage())
.build();
}
private PageData toPageData(PageNode pageNode) {
return PageData.builder()
.height(pageNode.getHeight())
.width(pageNode.getWidth())
.number(pageNode.getNumber())
.footer(pageNode.getFooter().getId())
.header(pageNode.getHeader().getId())
.build();
}
private AtomicTextBlockData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {
return AtomicTextBlockData.builder()
.id(atomicTextBlock.getId())
.searchText(atomicTextBlock.getSearchText())
.start(atomicTextBlock.getBoundary().start())
.end(atomicTextBlock.getBoundary().end())
.lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
.stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
.positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
.build();
}
private float[][] toPrimitiveFloatMatrix(List<Rectangle2D> positions) {
float[][] positionMatrix = new float[positions.size()][];
for (int i = 0; i < positions.size(); i++) {
float[] singlePositions = new float[4];
singlePositions[0] = (float) positions.get(i).getMinX();
singlePositions[1] = (float) positions.get(i).getMinY();
singlePositions[2] = (float) positions.get(i).getWidth();
singlePositions[3] = (float) positions.get(i).getHeight();
positionMatrix[i] = singlePositions;
}
return positionMatrix;
}
private int[] toPrimitiveIntArray(List<Integer> list) {
int[] array = new int[list.size()];
for (int i = 0; i < list.size(); i++) {
array[i] = list.get(i);
}
return array;
}
}

75
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java
View File

@ -11,6 +11,7 @@ import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -22,16 +23,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.Section; import com.iqser.red.service.redaction.v1.server.classification.model.Section;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
@ -51,13 +51,7 @@ public class DocumentGraphFactory {
public DocumentGraph buildDocumentGraph(Document document) { public DocumentGraph buildDocumentGraph(Document document) {
Context context = new Context(new TableOfContents(), Context context = new Context(new TableOfContents(), new LinkedList<>(), new LinkedList<>(), new AtomicInteger(0), new AtomicLong(0));
new LinkedList<>(),
new LinkedList<>(),
new LinkedList<>(),
new LinkedList<>(),
new AtomicInteger(0),
new AtomicInteger(0));
context.pages.addAll(document.getPages().stream().map(this::buildPage).toList()); context.pages.addAll(document.getPages().stream().map(this::buildPage).toList());
@ -79,16 +73,15 @@ public class DocumentGraphFactory {
private void addSections(Document document, Context context) { private void addSections(Document document, Context context) {
for (int sectionIdx = 0; sectionIdx < document.getSections().size(); sectionIdx++) { for (var section : document.getSections()) {
addSection(document.getSections().get(sectionIdx), sectionIdx, context); addSection(section, context);
} }
} }
private void addSection(Section section, int sectionIdx, Context context) { private void addSection(Section section, Context context) {
SectionNode sectionEntity = SectionNode.builder() SectionNode sectionEntity = SectionNode.builder()
.id(sectionIdx)
.entities(new LinkedList<>()) .entities(new LinkedList<>())
.pages(new LinkedList<>()) .pages(new LinkedList<>())
.paragraphs(new LinkedList<>()) .paragraphs(new LinkedList<>())
@ -99,15 +92,19 @@ public class DocumentGraphFactory {
context.sections().add(sectionEntity); context.sections().add(sectionEntity);
List<AbstractTextContainer> pageBlocks = new ArrayList<>(section.getPageBlocks()); List<AbstractTextContainer> pageBlocks = new ArrayList<>(section.getPageBlocks());
PageNode page = getPage(section.getPageBlocks().get(0).getPage(), context);
sectionEntity.getPages().add(page);
page.getMainBody().add(sectionEntity);
if (pageBlocks.get(0) instanceof TextBlock) { if (pageBlocks.get(0) instanceof TextBlock) {
sectionEntity.setHeadline(buildAtomicTextBlock(((TextBlock) pageBlocks.get(0)).getSequences(), sectionEntity, context)); sectionEntity.setHeadline(buildAtomicTextBlock(((TextBlock) pageBlocks.get(0)).getSequences(), sectionEntity, context));
sectionEntity.setNumberOnPage(((TextBlock) pageBlocks.get(0)).getIndexOnPage());
pageBlocks.remove(0); pageBlocks.remove(0);
sectionEntity.getPages().add(getPage(section.getPageBlocks().get(0).getPage(), context));
} else { } else {
sectionEntity.setNumberOnPage(1);
sectionEntity.setHeadline(emptyTextBlock(sectionEntity, context)); sectionEntity.setHeadline(emptyTextBlock(sectionEntity, context));
} }
String sectionId = context.tableOfContents.createNewEntryAndReturnId(TableOfContents.SECTION, buildSummary(sectionEntity.getHeadline()), sectionEntity); String sectionId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, buildSummary(sectionEntity.getHeadline()), sectionEntity);
sectionEntity.setTocId(sectionId); sectionEntity.setTocId(sectionId);
int paragraphIdx = 0; int paragraphIdx = 0;
@ -117,7 +114,7 @@ public class DocumentGraphFactory {
addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx, context); addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx, context);
paragraphIdx++; paragraphIdx++;
} else if (abstractTextContainer instanceof Table) { } else if (abstractTextContainer instanceof Table) {
addTable(sectionEntity, (Table) abstractTextContainer, tableIdx, context); //addTable(sectionEntity, (Table) abstractTextContainer, tableIdx, context);
tableIdx++; tableIdx++;
} }
@ -130,12 +127,11 @@ public class DocumentGraphFactory {
PageNode page = getPage(table.getPage(), context); PageNode page = getPage(table.getPage(), context);
TableNode tableEntity = TableNode.builder().id(tableIdx).tableOfContents(context.tableOfContents()).pages(new LinkedList<>()).parentSection(sectionEntity).build(); TableNode tableEntity = TableNode.builder().id(tableIdx).tableOfContents(context.tableOfContents()).pages(new LinkedList<>()).parentSection(sectionEntity).build();
sectionEntity.getTables().add(tableEntity); sectionEntity.getTables().add(tableEntity);
page.getMainBody().add(tableEntity);
if (!page.getMainBody().contains(sectionEntity)) { if (!page.getMainBody().contains(sectionEntity)) {
sectionEntity.getPages().add(page); sectionEntity.getPages().add(page);
page.getMainBody().add(sectionEntity);
} }
page.getMainBody().add(tableEntity);
} }
@ -143,19 +139,18 @@ public class DocumentGraphFactory {
private void addParagraph(SectionNode sectionEntity, TextBlock originalTextBlock, int paragraphIdx, Context context) { private void addParagraph(SectionNode sectionEntity, TextBlock originalTextBlock, int paragraphIdx, Context context) {
PageNode page = getPage(originalTextBlock.getPage(), context); PageNode page = getPage(originalTextBlock.getPage(), context);
ParagraphNode paragraph = ParagraphNode.builder().id(paragraphIdx).numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build(); ParagraphNode paragraph = ParagraphNode.builder().numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build();
sectionEntity.getParagraphs().add(paragraph); sectionEntity.getParagraphs().add(paragraph);
page.getMainBody().add(paragraph);
if (!page.getMainBody().contains(sectionEntity)) { if (!page.getMainBody().contains(sectionEntity)) {
sectionEntity.getPages().add(page); sectionEntity.getPages().add(page);
page.getMainBody().add(sectionEntity);
} }
page.getMainBody().add(paragraph);
var textBlock = buildAtomicTextBlock(originalTextBlock.getSequences(), paragraph, context); var textBlock = buildAtomicTextBlock(originalTextBlock.getSequences(), paragraph, context);
paragraph.setAtomicTextBlock(textBlock); paragraph.setAtomicTextBlock(textBlock);
String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), TableOfContents.PARAGRAPH, buildSummary(textBlock), paragraph); String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), NodeType.PARAGRAPH, buildSummary(textBlock), paragraph);
paragraph.setTocId(tocId); paragraph.setTocId(tocId);
} }
@ -195,9 +190,7 @@ public class DocumentGraphFactory {
private void addFooter(List<TextBlock> textBlocks, Context context) { private void addFooter(List<TextBlock> textBlocks, Context context) {
PageNode page = getPage(textBlocks.get(0).getPage(), context); PageNode page = getPage(textBlocks.get(0).getPage(), context);
FooterNode footer = FooterNode.builder().page(page).build(); AtomicTextBlock footer = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), page, context);
AtomicTextBlock atomicTextBlock = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), footer, context);
footer.setAtomicTextBlock(atomicTextBlock);
page.setFooter(footer); page.setFooter(footer);
} }
@ -205,9 +198,7 @@ public class DocumentGraphFactory {
public void addHeader(List<TextBlock> textBlocks, Context context) { public void addHeader(List<TextBlock> textBlocks, Context context) {
PageNode page = getPage(textBlocks.get(0).getPage(), context); PageNode page = getPage(textBlocks.get(0).getPage(), context);
HeaderNode header = HeaderNode.builder().page(page).build(); AtomicTextBlock header = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), page, context);
AtomicTextBlock atomicTextBlock = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), header, context);
header.setAtomicTextBlock(atomicTextBlock);
page.setHeader(header); page.setHeader(header);
} }
@ -215,24 +206,18 @@ public class DocumentGraphFactory {
private void addEmptyFooter(int pageIndex, Context context) { private void addEmptyFooter(int pageIndex, Context context) {
PageNode page = getPage(pageIndex, context); PageNode page = getPage(pageIndex, context);
FooterNode footer = FooterNode.builder().page(page).build(); page.setFooter(emptyTextBlock(page, context));
AtomicTextBlock atomicTextBlock = emptyTextBlock(footer, context);
footer.setAtomicTextBlock(atomicTextBlock);
page.setFooter(footer);
} }
private void addEmptyHeader(int pageIndex, Context context) { private void addEmptyHeader(int pageIndex, Context context) {
PageNode page = getPage(pageIndex, context); PageNode page = getPage(pageIndex, context);
HeaderNode header = HeaderNode.builder().page(page).build(); page.setHeader(emptyTextBlock(page, context));
AtomicTextBlock atomicTextBlock = emptyTextBlock(header, context);
header.setAtomicTextBlock(atomicTextBlock);
page.setHeader(header);
} }
private AtomicTextBlock emptyTextBlock(DocumentNode parent, Context context) { private AtomicTextBlock emptyTextBlock(DocumentGraphNode parent, Context context) {
return AtomicTextBlock.builder() return AtomicTextBlock.builder()
.id(context.textBlockIdx.getAndIncrement()) .id(context.textBlockIdx.getAndIncrement())
@ -252,7 +237,7 @@ public class DocumentGraphFactory {
return " probably a table"; return " probably a table";
} }
String[] words = textBlock.getFirstLine().split(" "); String[] words = textBlock.getFirstLine().toString().split(" ");
int bound = Math.min(words.length, 4); int bound = Math.min(words.length, 4);
List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound)); List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
@ -279,7 +264,7 @@ public class DocumentGraphFactory {
} }
private AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, DocumentNode parent, Context context) { private AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, DocumentGraphNode parent, Context context) {
SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences); SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
int offset = context.stringOffset().getAndAdd(searchTextWithTextPositionModel.getSearchText().length()); int offset = context.stringOffset().getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
@ -312,13 +297,7 @@ public class DocumentGraphFactory {
record Context( record Context(
TableOfContents tableOfContents, TableOfContents tableOfContents, List<PageNode> pages, List<SectionNode> sections, AtomicInteger stringOffset, AtomicLong textBlockIdx) {
List<PageNode> pages,
List<SectionNode> sections,
List<HeaderNode> headers,
List<FooterNode> footers,
AtomicInteger stringOffset,
AtomicInteger textBlockIdx) {
} }

171
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java Normal file
View File

@ -0,0 +1,171 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.lang3.NotImplementedException;
import org.springframework.stereotype.Service;
import com.google.common.primitives.Ints;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
@Service
public class DocumentGraphMapper {
public DocumentGraph toDocumentGraph(DocumentData documentData) {
Context context = new Context(documentData, new TableOfContents(), new LinkedList<>(), new LinkedList<>(), documentData.getAtomicTextBlocks());
context.pages.addAll(documentData.getPages().stream().map(pageData -> buildPage(pageData, context)).toList());
buildNodesFromTableOfContents("", context);
DocumentGraph documentGraph= DocumentGraph.builder()
.numberOfPages(documentData.getPages().size())
.pages(context.pages)
.sections(context.sections)
.tableOfContents(context.tableOfContents)
.build();
documentGraph.setText(documentGraph.buildTextBlock());
return documentGraph;
}
private void buildNodesFromTableOfContents(String currentTocId, Context context) {
List <TableOfContentsData.EntryData> entries;
if(currentTocId.equals("")) {
entries = context.documentData().getTableOfContents().getEntries();
} else {
entries = context.documentData().getTableOfContents().get(currentTocId).subEntries();
}
for (TableOfContentsData.EntryData entryData : entries) {
switch (entryData.type()) {
case SECTION -> buildSection(entryData, currentTocId, context);
case PARAGRAPH -> buildParagraph(entryData, currentTocId, context);
default -> throw new NotImplementedException("Not yet implemented for type " + entryData.type());
}
}
}
private void buildSection(TableOfContentsData.EntryData entryData, String currentTocId, Context context) {
SectionNode section = SectionNode.builder()
.entities(new LinkedList<>())
.pages(new LinkedList<>())
.paragraphs(new LinkedList<>())
.tables(new LinkedList<>())
.subSections(new LinkedList<>())
.tableOfContents(context.tableOfContents())
.numberOnPage(entryData.numberOnPage())
.build();
context.sections().add(section);
section.setHeadline(toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(entryData.atomicTextBlock())), section));
if (!currentTocId.equals("")) {
SectionNode parent = (SectionNode) context.tableOfContents().getEntryById(currentTocId).node();
section.setParentSection(parent);
parent.getSubSections().add(section);
}
PageNode page = getPage(entryData.page(), context);
page.getMainBody().add(section);
section.getPages().add(page);
String sectionId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, buildSummary(section.getHeadline()), section);
section.setTocId(sectionId);
buildNodesFromTableOfContents(sectionId, context);
}
private void buildParagraph(TableOfContentsData.EntryData entryData, String currentTocId, Context context) {
PageNode page = getPage(entryData.page(), context);
SectionNode parentSection = (SectionNode) context.tableOfContents().getEntryById(currentTocId).node();
ParagraphNode paragraph = ParagraphNode.builder().numberOnPage(entryData.numberOnPage()).page(page).parentSection(parentSection).build();
AtomicTextBlock atomicTextBlock = toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(entryData.atomicTextBlock())), paragraph);
paragraph.setAtomicTextBlock(atomicTextBlock);
if (!page.getMainBody().contains(parentSection)) {
parentSection.getPages().add(page);
}
page.getMainBody().add(paragraph);
String tocId = context.tableOfContents.createNewChildEntryAndReturnId(currentTocId, NodeType.PARAGRAPH, buildSummary(atomicTextBlock), paragraph);
paragraph.setTocId(tocId);
}
private PageNode buildPage(PageData p, Context context) {
PageNode page = PageNode.builder().height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
AtomicTextBlock header = toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(p.getHeader())), page);
AtomicTextBlock footer = toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(p.getFooter())), page);
page.setHeader(header);
page.setFooter(footer);
return page;
}
private static String buildSummary(com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock) {
if (textBlock == null) {
return " probably a table";
}
String[] words = textBlock.getFirstLine().toString().split(" ");
int bound = Math.min(words.length, 4);
List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
return String.join(" ", list);
}
private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, DocumentGraphNode parent) {
return AtomicTextBlock.builder()
.id(atomicTextBlockData.getId())
.searchText(atomicTextBlockData.getSearchText())
.boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
.lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks()))
.positions(Arrays.stream(atomicTextBlockData.getPositions())
.map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
.toList())
.stringIdxToPositionIdx(Ints.asList(atomicTextBlockData.getStringIdxToPositionIdx()))
.parent(parent)
.build();
}
private PageNode getPage(Long pageIndex, Context context) {
return context.pages.stream()
.filter(page -> page.getNumber() == toIntExact(pageIndex))
.findFirst()
.orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex)));
}
record Context(DocumentData documentData, TableOfContents tableOfContents, List<PageNode> pages, List<SectionNode> sections, List<AtomicTextBlockData> atomicTextBlockData) {
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java
View File

@ -9,7 +9,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
public class RegexMatcher { public class RegexMatcher {
public static boolean anyMatch(String regexPattern, CharSequence searchText) { public static boolean anyMatch(CharSequence searchText, String regexPattern) {
var pattern = Patterns.getCompiledPattern(regexPattern, false); var pattern = Patterns.getCompiledPattern(regexPattern, false);
return pattern.matcher(searchText).find(); return pattern.matcher(searchText).find();

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java
View File

@ -93,13 +93,13 @@ public class AbstractTestWithDictionaries {
protected final static String TEST_FILE_ID = "123"; protected final static String TEST_FILE_ID = "123";
@Autowired @Autowired
private StorageService storageService; protected StorageService storageService;
@MockBean @MockBean
private DictionaryClient dictionaryClient; protected DictionaryClient dictionaryClient;
@MockBean @MockBean
private RulesClient rulesClient; protected RulesClient rulesClient;
private final Map<String, List<String>> dictionary = new HashMap<>(); private final Map<String, List<String>> dictionary = new HashMap<>();
private final Map<String, List<String>> dossierDictionary = new HashMap<>(); private final Map<String, List<String>> dossierDictionary = new HashMap<>();

15
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java
View File

@ -22,12 +22,14 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.model.FileAttribute; import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.services.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
@ -48,11 +50,16 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
@Autowired @Autowired
private DictionaryService dictionaryService; private DictionaryService dictionaryService;
@Autowired
private DocumentDataMapper documentDataMapper;
@Qualifier("kieContainer") @Qualifier("kieContainer")
@Autowired @Autowired
private KieContainer kieContainer; private KieContainer kieContainer;
@Test @Test
@SneakyThrows @SneakyThrows
public void testDroolsOnDocumentGraph() { public void testDroolsOnDocumentGraph() {
@ -227,17 +234,17 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
.forEach(foundEntities::add); .forEach(foundEntities::add);
}); });
documentGraph.getPages().forEach( page -> { documentGraph.getPages().forEach( page -> {
TextBlock textBlock = page.getHeader().buildTextBlock(); TextBlock textBlock = page.getHeader();
searchImplementation.getMatches(textBlock, textBlock.getBoundary().start()) searchImplementation.getMatches(textBlock, textBlock.getBoundary().start())
.stream() .stream()
.map(bounds -> page.getHeader().createAndAddEntity(bounds, type, entityType)) .map(bounds -> page.createAndAddEntity(bounds, type, entityType))
.forEach(foundEntities::add); .forEach(foundEntities::add);
}); });
documentGraph.getPages().forEach( page -> { documentGraph.getPages().forEach( page -> {
TextBlock textBlock = page.getFooter().buildTextBlock(); TextBlock textBlock = page.getFooter();
searchImplementation.getMatches(textBlock, textBlock.getBoundary().start()) searchImplementation.getMatches(textBlock, textBlock.getBoundary().start())
.stream() .stream()
.map(bounds -> page.getFooter().createAndAddEntity(bounds, type, entityType)) .map(bounds -> page.createAndAddEntity(bounds, type, entityType))
.forEach(foundEntities::add); .forEach(foundEntities::add);
}); });
/* /*

55
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphMappingTest.java Normal file
View File

@ -0,0 +1,55 @@
package com.iqser.red.service.redaction.v1.server;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.services.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import lombok.SneakyThrows;
public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
@Autowired
private DocumentGraphFactory documentGraphFactory;
@Autowired
private PdfSegmentationService segmentationService;
@Autowired
private DocumentDataMapper documentDataMapper;
@Autowired
private DocumentGraphMapper documentGraphMapper;
@Test
@SneakyThrows
public void testGraphMapping() {
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
prepareStorage(filename + ".pdf");
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
DocumentData documentData = documentDataMapper.toDocumentData(document);
storageService.storeJSONObject(filename + ".json", documentData);
DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData);
assert document.toString().equals(newDocumentGraph.toString());
assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString());
for (int pageIndex = 1; pageIndex < document.getNumberOfPages(); pageIndex++) {
var page = document.getPages().get(pageIndex);
var newPage = document.getPages().get(pageIndex);
assert page.toString().equals(newPage.toString());
}
}
}

126
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl
View File

@ -19,8 +19,8 @@ rule "0: Expand CBI Authors with firstname initials"
when when
Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author")) Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
then then
section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+",dictionary);
section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+"); section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+",dictionary);
end end
@ -53,7 +53,7 @@ rule "4: Redact Author(s) cells in Tables with Author(s) header"
when when
Section(hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) Section(hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -61,7 +61,7 @@ rule "5: Redact Author cells in Tables with Author header"
when when
Section(hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) Section(hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -69,7 +69,7 @@ rule "6: Redact and recommand Authors in Tables with Vertebrate study Y/N header
when when
Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")) Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
then then
section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -77,8 +77,8 @@ rule "7: Redact if CTL/* or BL/* was found"
when when
Section(searchText.contains("CTL/") || searchText.contains("BL/")) Section(searchText.contains("CTL/") || searchText.contains("BL/"))
then then
section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" ); section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002", dictionary);
section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" ); section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002", dictionary);
end end
@ -86,7 +86,7 @@ rule "8: Redact and add recommendation for et al. author"
when when
Section(searchText.contains("et al")) Section(searchText.contains("et al"))
then then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)",dictionary);
end end
@ -130,7 +130,7 @@ rule "13: Redact Emails by RegEx"
when when
Section(searchText.contains("@")) Section(searchText.contains("@"))
then then
section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -153,25 +153,25 @@ rule "14: Redact contact information"
|| text.contains("Phone No.") || text.contains("Phone No.")
|| text.contains("European contact:")) || text.contains("European contact:"))
then then
section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("European contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -179,25 +179,25 @@ rule "15: Redact contact information if applicant is found"
when when
Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")) Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
then then
section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("European contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -205,17 +205,17 @@ rule "16: Redact contact information if Producer is found"
when when
Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")) Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
then then
section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -223,7 +223,7 @@ rule "17: Redact AUTHOR(S)"
when when
Section(searchText.contains("AUTHOR(S):")) Section(searchText.contains("AUTHOR(S):"))
then then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -231,7 +231,7 @@ rule "18: Redact PERFORMING LABORATORY"
when when
Section(searchText.contains("PERFORMING LABORATORY:")) Section(searchText.contains("PERFORMING LABORATORY:"))
then then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -239,7 +239,7 @@ rule "19: Redact On behalf of Sequani Ltd.:"
when when
Section(searchText.contains("On behalf of Sequani Ltd.: Name Title")) Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
then then
section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -247,7 +247,7 @@ rule "20: Redact On behalf of Syngenta Ltd.:"
when when
Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title")) Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
then then
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002"); section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -257,7 +257,7 @@ rule "21: Purity Hint"
when when
Section(searchText.toLowerCase().contains("purity")) Section(searchText.toLowerCase().contains("purity"))
then then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
end end

4
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl
View File

@ -23,7 +23,7 @@ global DocumentGraph document
rule "1: Redact CBI_author" rule "1: Redact CBI_author"
when when
FileAttribute(label == "Vertebrate Study" && (value.toLowerCase() == "yes")) FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
entity: EntityNode(type == "CBI_author") entity: EntityNode(type == "CBI_author")
then then
entity.setRedact(true); entity.setRedact(true);
@ -33,7 +33,7 @@ rule "1: Redact CBI_author"
rule "2: do not redact genitive CBI_author" rule "2: do not redact genitive CBI_author"
when when
entity: EntityNode(type == "CBI_author", anyMatch("['’’'ʼˈ´`‘′ʻ’']s", textAfter), redact == true) entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redact == true)
then then
entity.setRedact(false); entity.setRedact(false);
entity.setEntityType(EntityType.FALSE_POSITIVE); entity.setEntityType(EntityType.FALSE_POSITIVE);

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/headlines.drl
View File

@ -9,5 +9,5 @@ rule "1: Find headlines"
when when
Section(text.length() > 1) Section(text.length() > 1)
then then
section.redactHeadline("headline", 1, "Headline found", "n-a."); section.redactHeadline("headline", 1, "Headline found", "n-a.",dictionary);
end end

383
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_rules.drl Normal file
View File

@ -0,0 +1,383 @@
package drools
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.redaction.model.*
global RedactionServiceSettings redactionServiceSettings;
global Dictionary dictionary;
// --------------------------------------- AI rules -------------------------------------------------------------------
rule "-1: find entities from dictionary"
salience 100
no-loop true
when
section: Section()
then
section.findDictionaryEntities(dictionary, redactionServiceSettings);
update(section);
end
rule "0: Add CBI_author from ai"
when
section: Section(aiMatchesType("CBI_author"))
then
section.addAiEntities("CBI_author", "CBI_author", dictionary);
end
rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
when
section: Section(aiMatchesType("ORG"))
then
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
end
rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
when
section: Section(aiMatchesType("STREET"))
then
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
end
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
when
section: Section(aiMatchesType("CITY"))
then
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary);
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "1: Redact CBI Authors (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
then
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "2: Redact CBI Authors (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
then
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "3: Redact not CBI Address (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
then
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
section.ignoreRecommendations("CBI_address");
end
rule "4: Redact CBI Address (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
then
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "5: Do not redact genitive CBI_author"
when
section: Section(matchesType("CBI_author"))
then
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary);
end
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "et al")
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
end
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "et al")
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
end
rule "16: Add recommendation for Addresses in Test Organism sections"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText (contains "Species:" && contains "Source:"))
then
section.recommendLineAfter("Source:", "CBI_address");
end
rule "17: Add recommendation for Addresses in Test Animals sections"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText (contains "Species" && contains "Source") )
then
section.recommendLineAfter("Source", "CBI_address");
end
/*
rule "18: Do not redact Names and Addresses if Published Information found"
when
section: Section(matchesType("published_information"))
then
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
end
*/
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
then
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
then
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "21: Redact Emails by RegEx (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "@")
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
end
rule "22: Redact Emails by RegEx (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "@")
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "23: Redact contact information (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study", "Yes") &&
(text (contains "Contact point:" ||
contains "Contact:" ||
contains "Alternative contact:" ||
(contains "No:" && contains "Fax") ||
(contains "Contact:" && contains "Tel.:") ||
contains "European contact:")))
then
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end
rule "24: Redact contact information (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") &&
(text (contains "Contact point:"
|| contains "Contact:"
|| contains"Alternative contact:"
|| (contains "No:" && contains "Fax")
|| (contains "Contact:" && contains "Tel.:")
|| contains "European contact:"
)))
then
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text contains "Contact"
|| contains "Telephone"
|| contains "Phone"
|| contains "Fax"
|| contains "Tel"
|| contains "Ter"
|| contains "Mobile"
|| contains "Fel"
|| contains "Fer"
))
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
end
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text contains "Contact"
|| contains "Telephone"
|| contains "Phone"
|| contains "Fax"
|| contains "Tel"
|| contains "Ter"
|| contains "Mobile"
|| contains "Fel"
|| contains "Fer"
))
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
end
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& (searchText contains "AUTHOR(S):"
&& contains "COMPLETION DATE:"
&& not contains "STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
end
rule "28: Redact AUTHOR(S) (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& (searchText contains "AUTHOR(S):"
&& contains "COMPLETION DATE:"
&& not contains "STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
end
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& (searchText contains "AUTHOR(S):"
&& contains "STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
end
rule "30: Redact AUTHOR(S) (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText contains "AUTHOR(S):"
&& contains "STUDY COMPLETION DATE:"
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
end
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText contains "PERFORMING LABORATORY:"
)
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
end
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText contains "PERFORMING LABORATORY:")
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
end
rule "33: Redact study director abbreviation"
when
section: Section(searchText contains "KATH" || contains "BECH" || contains "KML")
then
section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "34: Purity Hint"
when
section: Section(searchText.toLowerCase() contains "purity")
then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
end
rule "35: Redact signatures (Non vertebrate study)"
when
section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "36: Redact signatures (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "43: Redact Logos (Vertebrate study)"
when
section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
then
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end

63
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1,67 +1,40 @@
package drools package drools
import static java.lang.String.format; import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch;
import java.util.List; import java.util.List;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.HashSet; import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.document.graph.*
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.Engine;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import java.util.Set; import java.util.Set;
global DocumentGraph document
global Dictionary dictionary
query "getEntities"
$result: Entity()
end
// --------------------------------------- AI rules ------------------------------------------------------------------- rule "1: Redact CBI_author"
rule "0: find entities from entry dictionary"
salience 100
when when
$section: Section() FileAttribute(label == "Vertebrate Study" && (value.toLowerCase() == "yes"))
$paragraph: Paragraph(sectionNumber == $section.sectionNumber) entity: EntityNode(type == "CBI_author")
$entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition())
then then
List<EntityPositionSequence> eps = new LinkedList<>(); entity.setRedact(true);
eps.add($entityPositionSequence); update(entity)
Set<Engine> engineSet = new HashSet<>();
engineSet.add(Engine.DICTIONARY);
Set<Entity> empty = new HashSet<>();
insert(new Entity("CBI_author",
"custom",
true,
format("Found in Dictionary: %s", "custom"),
eps,
$section.getHeadline(),
0,
$section.getSectionNumber(),
$paragraph.getParagraphNumber(),
"",
true,
"",
"",
-1,
-1,
false,
engineSet,
empty,
EntityType.ENTITY
));
end end
/* rule "2: do not redact genitive CBI_author"
rule "44: Don't redact Author if contained in paragraph with published information"
when
$section: Section()
$paragraph: Paragraph($section.sectionNumber == sectionNumber)
$authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber()) when
$publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber()) entity: EntityNode(type == "CBI_author", anyMatch("['’’'ʼˈ´`‘′ʻ’']s", textAfter))
then then
$authorEntity.setRedaction(false); entity.setRedact(false);
entity.setEntityType(EntityType.FALSE_POSITIVE);
update(entity)
end end
*/

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl Normal file
View File

@ -0,0 +1 @@
rules.drl

124
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl
View File

@ -11,28 +11,28 @@ rule "0: Add CBI_author from ai"
when when
Section(aiMatchesType("CBI_author")) Section(aiMatchesType("CBI_author"))
then then
section.addAiEntities("CBI_author", "CBI_author"); section.addAiEntities("CBI_author", "CBI_author",dictionary);
end end
rule "0: Combine address parts from ai to CBI_address (org is mandatory)" rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
when when
Section(aiMatchesType("ORG")) Section(aiMatchesType("ORG"))
then then
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
end end
rule "0: Combine address parts from ai to CBI_address (street is mandatory)" rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
when when
Section(aiMatchesType("STREET")) Section(aiMatchesType("STREET"))
then then
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
end end
rule "0: Combine address parts from ai to CBI_address (city is mandatory)" rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
when when
Section(aiMatchesType("CITY")) Section(aiMatchesType("CITY"))
then then
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false); section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary);
end end
@ -73,7 +73,7 @@ rule "5: Do not redact genitive CBI_author"
when when
Section(matchesType("CBI_author")) Section(matchesType("CBI_author"))
then then
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary);
end end
@ -81,14 +81,14 @@ rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -96,14 +96,14 @@ rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -111,14 +111,14 @@ rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N heade
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then then
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then then
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
/* Syngenta specific laboratory rule */ /* Syngenta specific laboratory rule */
@ -133,14 +133,14 @@ rule "14: Redact and add recommendation for et al. author (Non vertebrate study)
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
rule "15: Redact and add recommendation for et al. author (Vertebrate study)" rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -190,14 +190,14 @@ rule "21: Redact Emails by RegEx (Non vertebrate study)"
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
rule "22: Redact Emails by RegEx (Vertebrate study)" rule "22: Redact Emails by RegEx (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -224,25 +224,25 @@ rule "23: Redact contact information (Non vertebrate study)"
|| text.contains("European contact:") || text.contains("European contact:")
)) ))
then then
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
rule "24: Redact contact information (Vertebrate study)" rule "24: Redact contact information (Vertebrate study)"
@ -268,25 +268,25 @@ rule "24: Redact contact information (Vertebrate study)"
|| text.contains("European contact:") || text.contains("European contact:")
)) ))
then then
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -305,7 +305,7 @@ rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
|| text.contains("Fer") || text.contains("Fer")
)) ))
then then
section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
@ -323,7 +323,7 @@ rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
|| text.contains("Fer") || text.contains("Fer")
)) ))
then then
section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -335,7 +335,7 @@ rule "27: Redact AUTHOR(S) (Non vertebrate study)"
&& !searchText.contains("STUDY COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:")
) )
then then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
rule "28: Redact AUTHOR(S) (Vertebrate study)" rule "28: Redact AUTHOR(S) (Vertebrate study)"
@ -346,7 +346,7 @@ rule "28: Redact AUTHOR(S) (Vertebrate study)"
&& !searchText.contains("STUDY COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:")
) )
then then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -357,7 +357,7 @@ rule "29: Redact AUTHOR(S) (Non vertebrate study)"
&& searchText.contains("STUDY COMPLETION DATE:") && searchText.contains("STUDY COMPLETION DATE:")
) )
then then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
rule "30: Redact AUTHOR(S) (Vertebrate study)" rule "30: Redact AUTHOR(S) (Vertebrate study)"
@ -367,7 +367,7 @@ rule "30: Redact AUTHOR(S) (Vertebrate study)"
&& searchText.contains("STUDY COMPLETION DATE:") && searchText.contains("STUDY COMPLETION DATE:")
) )
then then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -377,7 +377,7 @@ rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
&& searchText.contains("PERFORMING LABORATORY:") && searchText.contains("PERFORMING LABORATORY:")
) )
then then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
end end
@ -386,7 +386,7 @@ rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:")) && searchText.contains("PERFORMING LABORATORY:"))
then then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -396,7 +396,7 @@ rule "33: Purity Hint"
when when
Section(searchText.toLowerCase().contains("purity")) Section(searchText.toLowerCase().contains("purity"))
then then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
end end

12
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -14,15 +14,21 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import java.util.Set; import java.util.Set;
global DocumentGraph document; global DocumentGraph document
rule "dummy"
when
then
System.out.println("This actually works");
end
rule "1: Redact CBI_author" rule "1: Redact CBI_author"
when when
FileAttribute(label == "Vertebrate Study" && (value == "Yes" || value == "Y" || value == "y")) //FileAttribute(label == "Vertebrate Study" && (value == "Yes"))
$entity: Entity2(type == "CBI_author") $entity: EntityNode(type == "CBI_author")
then then
System.out.println("Entity found");
modify($entity){ modify($entity){
$entity.setRedact(true) $entity.setRedact(true)
} }

242
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -10,7 +10,7 @@ rule "0: Add CBI_author from ai"
when when
Section(aiMatchesType("CBI_author")) Section(aiMatchesType("CBI_author"))
then then
section.addAiEntities("CBI_author", "CBI_author"); section.addAiEntities("CBI_author", "CBI_author",dictionary);
end end
@ -18,7 +18,7 @@ rule "0: Combine ai types CBI_author from ai"
when when
Section(aiMatchesType("ORG")) Section(aiMatchesType("ORG"))
then then
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
end end
@ -59,7 +59,7 @@ rule "5: Do not redact genitive CBI_author"
when when
Section(matchesType("CBI_author")) Section(matchesType("CBI_author"))
then then
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary);
end end
@ -67,7 +67,7 @@ rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -75,7 +75,7 @@ rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate stud
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -83,7 +83,7 @@ rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -91,7 +91,7 @@ rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then then
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -99,7 +99,7 @@ rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N heade
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then then
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -107,7 +107,7 @@ rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N heade
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then then
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -115,7 +115,7 @@ rule "13: Redact addresses that start with BL or CTL"
when when
Section(searchText.contains("BL") || searchText.contains("CT")) Section(searchText.contains("BL") || searchText.contains("CT"))
then then
section.redactNotAndRecommendByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address", 13, "Laboratory for vertebrate studies found"); section.redactNotAndRecommendByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address", 13, "Laboratory for vertebrate studies found",dictionary);
end end
@ -123,7 +123,7 @@ rule "14: Redact and add recommendation for et al. author (Non vertebrate study)
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -131,7 +131,7 @@ rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -183,7 +183,7 @@ rule "21: Redact Emails by RegEx (Non vertebrate study)"
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -191,7 +191,7 @@ rule "22: Redact Emails by RegEx (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -199,7 +199,7 @@ rule "23: Redact telephone numbers by RegEx (Non vertebrate study)"
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true)) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true))
then then
section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 23, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 23, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -207,7 +207,7 @@ rule "24: Redact telephone numbers by RegEx (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true)) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true))
then then
section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 24, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 24, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -238,25 +238,25 @@ rule "26: Redact contact information (Non vertebrate study)"
|| text.contains("Phone No.") || text.contains("Phone No.")
|| text.contains("European contact:"))) || text.contains("European contact:")))
then then
section.redactLineAfter("Contact point:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact point:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Email:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("e-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail address:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Alternative contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone No:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("Contact:", "Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("European contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -279,25 +279,25 @@ rule "27: Redact contact information (Vertebrate study)"
|| text.contains("Phone No.") || text.contains("Phone No.")
|| text.contains("European contact:"))) || text.contains("European contact:")))
then then
section.redactLineAfter("Contact point:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact point:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Email:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("e-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail address:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Alternative contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone No:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("Contact:", "Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("European contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -305,25 +305,25 @@ rule "28: Redact contact information if applicant is found (Non vertebrate study
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")))
then then
section.redactLineAfter("Contact point:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact point:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Email:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("e-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail address:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Alternative contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone No:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("Contact:", "Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("European contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -331,25 +331,25 @@ rule "29: Redact contact information if applicant is found (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")))
then then
section.redactLineAfter("Contact point:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact point:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Email:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Email:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("e-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("e-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail address:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail address:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Alternative contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Alternative contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone No:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone No:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("Contact:", "Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("Contact:", "Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("European contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("European contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -357,17 +357,17 @@ rule "30: Redact contact information if Producer is found (Non vertebrate study)
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")))
then then
section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -375,17 +375,17 @@ rule "31: Redact contact information if Producer is found (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")))
then then
section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("E-mail:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("E-mail:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Fax number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Fax number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Telephone number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Telephone number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Tel:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Tel:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactLineAfter("Phone No.", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLineAfter("Phone No.", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
section.redactBetween("No:", "Fax", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("No:", "Fax", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -393,7 +393,7 @@ rule "32: Redact AUTHOR(S) (Non vertebrate study)"
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:"))
then then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 32, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 32, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -401,7 +401,7 @@ rule "33: Redact AUTHOR(S) (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:"))
then then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 33, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 33, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -409,7 +409,7 @@ rule "34: Redact AUTHOR(S) (Non vertebrate study)"
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:"))
then then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 34, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 34, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
end end
@ -417,7 +417,7 @@ rule "35: Redact AUTHOR(S) (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:"))
then then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 35, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 35, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -425,7 +425,7 @@ rule "36: Redact PERFORMING LABORATORY (Non vertebrate study)"
when when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:")) Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:"))
then then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 36, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 36, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
section.redactNot("CBI_address", 36, "Performing laboratory found for non vertebrate study"); section.redactNot("CBI_address", 36, "Performing laboratory found for non vertebrate study");
end end
@ -434,7 +434,7 @@ rule "37: Redact PERFORMING LABORATORY (Vertebrate study)"
when when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:")) Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:"))
then then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 37, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 37, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
end end
@ -444,7 +444,7 @@ rule "50: Purity Hint"
when when
Section(searchText.toLowerCase().contains("purity")) Section(searchText.toLowerCase().contains("purity"))
then then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
end end