Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6369: Rules Refactor

Browse Source 
*added some new Rules and Methods to support them
*integrated local dictionary searches into ruleflow using agenda-groups
*Generalized methods to be able to execute on any SemanticNode
*made Documentgraph implement SemanticNode
This commit is contained in:
Kilian Schuettler 2023-03-22 19:05:33 +01:00 committed by Kilian Schuettler
parent 253e544724
commit b6307759fb
39 changed files with 1011 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java
View File

@ -20,7 +20,7 @@ public class DocumentDataMapper {
public DocumentData toDocumentData(DocumentGraph documentGraph) {
List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTextBlocksInOrder()
List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder()
.flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
.distinct()
.map(this::toAtomicTextBlockData)

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java
View File

@ -22,12 +22,12 @@ import com.iqser.red.service.redaction.v1.server.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
@ -44,7 +44,11 @@ public class DocumentGraphMapper {
buildNodesFromTableOfContents(Collections.emptyList(), context);
DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(documentData.getPages().size()).pages(context.pages).tableOfContents(context.tableOfContents).build();
DocumentGraph documentGraph = DocumentGraph.builder()
.numberOfPages(documentData.getPages().size())
.pages(new HashSet<>(context.pages))
.tableOfContents(context.tableOfContents)
.build();
documentGraph.setTextBlock(documentGraph.buildTextBlock());
return documentGraph;
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java
View File

@ -86,7 +86,7 @@ public class Boundary implements Comparable<Boundary> {
public List<Boundary> split(List<Integer> splitIndices) {
if(splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this));
}
List<Boundary> splitBoundaries = new LinkedList<>();
@ -100,6 +100,14 @@ public class Boundary implements Comparable<Boundary> {
}
public static Boundary merge(List<Boundary> boundaries) {
int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
int maxEnd = boundaries.stream().mapToInt(Boundary::end).max().orElseThrow(IllegalArgumentException::new);
return new Boundary(minStart, maxEnd);
}
@Override
public String toString() {
@ -109,6 +117,7 @@ public class Boundary implements Comparable<Boundary> {
@Override
public int compareTo(Boundary boundary) {
if (end < boundary.end() && start < boundary.start()) {
return -1;
}
@ -119,12 +128,14 @@ public class Boundary implements Comparable<Boundary> {
return 0;
}
@Override
public int hashCode() {
return toString().hashCode();
}
@Override
public boolean equals(Object object) {

41
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java
View File

@ -1,14 +1,18 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
@ -24,9 +28,9 @@ import lombok.extern.slf4j.Slf4j;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class DocumentGraph {
public class DocumentGraph implements SemanticNode {
List<PageNode> pages;
Set<PageNode> pages;
TableOfContents tableOfContents;
Integer numberOfPages;
TextBlock textBlock;
@ -34,7 +38,7 @@ public class DocumentGraph {
public TextBlock buildTextBlock() {
return streamTextBlocksInOrder().collect(new TextBlockCollector());
return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
}
@ -44,7 +48,7 @@ public class DocumentGraph {
}
public Stream<TextBlock> streamTextBlocksInOrder() {
public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
return streamAllNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock);
}
@ -56,6 +60,20 @@ public class DocumentGraph {
}
@Override
public List<Integer> getTocId() {
return Collections.emptyList();
}
@Override
public void setTocId(List<Integer> tocId) {
throw new UnsupportedOperationException("DocumentGraph is always the root of the Table of Contents");
}
private Stream<SemanticNode> streamAllNodes() {
return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node);
@ -68,4 +86,15 @@ public class DocumentGraph {
return tableOfContents.toString();
}
@Override
public Map<PageNode, Rectangle2D> getBBox() {
Map<PageNode, Rectangle2D> bBox = new HashMap<>();
for (PageNode page : pages) {
bBox.put(page, new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight()));
}
return bBox;
}
}

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageElement.java
View File

@ -1,14 +0,0 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import lombok.Data;
@Data
public class PageElement {
Map<Integer, RedRectangle2D> positionsPerPage;
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java
View File

@ -11,8 +11,11 @@ import java.util.List;
import java.util.stream.Stream;
import com.google.common.hash.Hashing;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import lombok.Builder;
import lombok.Data;
@ -29,6 +32,12 @@ public class TableOfContents {
}
public TextBlock buildTextBlock() {
return streamEntriesInOrder().map(Entry::node).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
}
public List<Integer> createNewEntryAndReturnId(NodeType nodeType, SemanticNode node) {
return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node);
@ -72,7 +81,7 @@ public class TableOfContents {
List<Integer> parentIds = getParentId(tocId);
if (parentIds.size() < 1) {
throw new UnsupportedOperationException(format("Node with tocId \"%s\" has no parent!", tocId));
throw new NotFoundException(format("Node with tocId \"%s\" has no parent!", tocId));
}
return getEntryById(parentIds);
}

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java
View File

@ -32,7 +32,6 @@ public class EntityNode {
.entityType(entityType)
.boundary(boundary)
.redaction(false)
.falsePositive(false)
.removed(false)
.ignored(false)
.resized(false)
@ -55,7 +54,6 @@ public class EntityNode {
// empty defaults
boolean redaction;
boolean falsePositive;
boolean removed;
boolean ignored;
boolean resized;
@ -117,17 +115,30 @@ public class EntityNode {
}
public boolean intersects(EntityNode entityNode) {
return this.boundary.intersects(entityNode.getBoundary());
}
public void addEngine(Engine engine) {
engines.add(engine);
}
public void addEngines(Set<Engine> engines) {
this.engines.addAll(engines);
}
public void addReference(EntityNode entityNode) {
references.add(entityNode);
}
@Override
public String toString() {
@ -157,6 +168,7 @@ public class EntityNode {
return Hashing.murmur3_128().hashString(toString(), StandardCharsets.UTF_8).hashCode();
}
@Override
public boolean equals(Object o) {

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph.entity;
import java.awt.geom.Rectangle2D;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.google.common.hash.Hashing;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
@ -17,7 +18,7 @@ import lombok.experimental.FieldDefaults;
public class EntityPosition {
PageNode pageNode;
Rectangle2D position;
List<Rectangle2D> rectanglePerLine;
public String getId() {
@ -29,8 +30,10 @@ public class EntityPosition {
@Override
public int hashCode() {
String string = String.valueOf(pageNode.getNumber()) + position.getX() + position.getY() + position.getWidth() + position.getHeight();
return Hashing.murmur3_128().hashString(string, StandardCharsets.UTF_8).hashCode();
StringBuilder sb = new StringBuilder();
sb.append(pageNode.getNumber());
rectanglePerLine.forEach(r -> sb.append(r.getX()).append(r.getY()).append(r.getWidth()).append(r.getHeight()));
return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode();
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java
View File

@ -8,7 +8,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
@ -30,7 +29,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class ImageNode extends PageElement implements SemanticNode {
public class ImageNode implements SemanticNode {
List<Integer> tocId;

47
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java
View File

@ -10,7 +10,6 @@ import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@ -28,16 +27,16 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Header;
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
@ -71,11 +70,7 @@ public class DocumentGraphFactory {
addSections(document, context);
addHeaderAndFooterToEachPage(document, context);
DocumentGraph documentGraph = DocumentGraph.builder()
.numberOfPages(context.pages.size())
.pages(context.pages.keySet().stream().sorted(Comparator.comparingInt(PageNode::getNumber)).toList())
.tableOfContents(context.tableOfContents)
.build();
DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(context.pages.size()).pages(context.pages.keySet()).tableOfContents(context.tableOfContents).build();
documentGraph.setTextBlock(documentGraph.buildTextBlock());
return documentGraph;
@ -193,28 +188,44 @@ public class DocumentGraphFactory {
tableCellNode.setTocId(tocId);
if (cell.getTextBlocks().isEmpty()) {
tableCellNode.setTerminal(false);
return;
}
tableCellNode.setTerminalTextBlock(context.textBlockFactory.emptyTextBlock(parentNode, context, page));
tableCellNode.setTerminal(true);
if (cell.getTextBlocks().size() == 1) {
} else if (cell.getTextBlocks().size() == 1) {
textBlock = context.textBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCellNode, context, page);
tableCellNode.setTerminalTextBlock(textBlock);
} else if (StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H")) {
tableCellNode.setTerminal(false);
tableCellNode.setTerminal(true);
} else if (firstTextBlockIsHeadline(cell)) {
addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractTextContainer) tb).toList(), Collections.emptyList(), context);
} else if (cell.getArea() < TABLE_CELL_MERGE_SIZE_THRESHOLD * page.getHeight() * page.getWidth()) {
tableCellNode.setTerminal(false);
} else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
List<TextPositionSequence> sequences = mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page);
tableCellNode.setTerminalTextBlock(textBlock);
tableCellNode.setTerminal(true);
} else {
tableCellNode.setTerminal(false);
cell.getTextBlocks().forEach(tb -> addParagraphOrHeadline(tableCellNode, tb, context));
tableCellNode.setTerminal(false);
}
}
private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, PageNode page) {
return cell.getArea() < TABLE_CELL_MERGE_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
}
private static boolean firstTextBlockIsHeadline(Cell cell) {
return StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H");
}
private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context) {
addParagraphOrHeadline(parentNode, originalTextBlock, context, Collections.emptyList());
@ -340,7 +351,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(pageIndex, context);
HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, context, 0, page);
AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header);
header.setTocId(tocId);
header.setTerminalTextBlock(textBlock);

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java
View File

@ -10,6 +10,8 @@ import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collector;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@ -52,6 +54,12 @@ public class RectangleTransformations {
}
public static Rectangle toRectangle(Rectangle2D rectangle2D, int pageNumber) {
return new Rectangle(new Point((float) rectangle2D.getMaxX(), (float) rectangle2D.getMaxY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
}
private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
@Override

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java
View File

@ -18,6 +18,7 @@ public class SearchTextWithTextPositionFactory {
public static final int HEIGHT_PADDING = 2;
public static SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
@ -79,8 +80,9 @@ public class SearchTextWithTextPositionFactory {
assert sb.length() == stringIdxToPositionIdx.size();
List<Rectangle2D> positions = sequences.stream()//
.flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToUserSpace(textPosition, sequence))).toList();
List<Rectangle2D> positions = sequences.stream()
.flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence)))
.toList();
return SearchTextWithTextPositionModel.builder()
.searchText(sb.toString())
@ -130,7 +132,7 @@ public class SearchTextWithTextPositionFactory {
}
private static Rectangle2D mapRedTextPositionToUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
private static Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
float textHeight = sequence.getTextHeight() + HEIGHT_PADDING;
Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(),

16
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextBlockFactory.java
View File

@ -16,7 +16,9 @@ public class TextBlockFactory {
AtomicInteger stringOffset;
AtomicLong textBlockIdx;
public TextBlockFactory() {
stringOffset = new AtomicInteger();
textBlockIdx = new AtomicLong();
@ -30,7 +32,11 @@ public class TextBlockFactory {
}
public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Integer numberOnPage, PageNode page) {
public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences,
SemanticNode parent,
DocumentGraphFactory.Context context,
Integer numberOnPage,
PageNode page) {
SearchTextWithTextPositionModel searchTextWithTextPositionModel = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
int offset = stringOffset.getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
@ -51,14 +57,14 @@ public class TextBlockFactory {
public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) {
return emptyTextBlock(parent, context, context.pages().get(page).getAndIncrement(), page);
return emptyTextBlock(parent, context.pages().get(page).getAndIncrement(), page);
}
public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, Integer numberOnPage, PageNode page) {
public AtomicTextBlock emptyTextBlock(SemanticNode parent, Integer numberOnPage, PageNode page) {
return AtomicTextBlock.builder()
.id(textBlockIdx.getAndIncrement())
.id(textBlockIdx.get())
.boundary(new Boundary(stringOffset.get(), stringOffset.get()))
.searchText("")
.lineBreaks(Collections.emptyList())
@ -70,6 +76,4 @@ public class TextBlockFactory {
.build();
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java
View File

@ -4,7 +4,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class FooterNode extends PageElement implements SemanticNode {
public class FooterNode implements SemanticNode {
List<Integer> tocId;
TextBlock terminalTextBlock;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java
View File

@ -4,7 +4,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class HeaderNode extends PageElement implements SemanticNode {
public class HeaderNode implements SemanticNode {
List<Integer> tocId;
TextBlock terminalTextBlock;
@ -44,6 +43,7 @@ public class HeaderNode extends PageElement implements SemanticNode {
return terminalTextBlock;
}
@Override
public String toString() {

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java
View File

@ -4,7 +4,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class HeadlineNode extends PageElement implements SemanticNode {
public class HeadlineNode implements SemanticNode {
List<Integer> tocId;
TextBlock terminalTextBlock;

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java
View File

@ -4,7 +4,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@ -20,7 +19,7 @@ import lombok.experimental.FieldDefaults;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class ParagraphNode extends PageElement implements SemanticNode {
public class ParagraphNode implements SemanticNode {
List<Integer> tocId;
TextBlock terminalTextBlock;
@ -42,9 +41,11 @@ public class ParagraphNode extends PageElement implements SemanticNode {
return terminalTextBlock;
}
@Override
public String toString() {
return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.toString();
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java
View File

@ -4,7 +4,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@ -24,7 +23,7 @@ import lombok.extern.slf4j.Slf4j;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class SectionNode extends PageElement implements SemanticNode {
public class SectionNode implements SemanticNode {
List<Integer> tocId;

96
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import static java.util.stream.Collectors.groupingBy;
import java.awt.geom.Rectangle2D;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -22,6 +21,7 @@ public interface SemanticNode {
/**
* Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity.
* So, for a Section all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity
* If the Node is Terminal, the TerminalTextBlock will be returned instead.
*
* @return TextBlock containing all AtomicTextBlocks that are located under this Node.
*/
@ -37,29 +37,42 @@ public interface SemanticNode {
Set<EntityNode> getEntities();
/**
* Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock
*
* @return Set of PageNodes this node appears on.
*/
default Set<PageNode> getPages() {
return buildTextBlock().getPages();
}
default PageNode getFirstPage() {
return getPages().stream().min(Comparator.comparingInt(PageNode::getNumber)).orElseThrow();
}
/**
* @return the TableOfContents of the Document this node belongs to
*/
TableOfContents getTableOfContents();
/**
* The id is a List of Integers uniquely identifying this node in the TableOfContents
*
* @return the TableOfContents ID
*/
List<Integer> getTocId();
/**
* This should only be used during graph construction
*
* @param tocId List of Integers
*/
void setTocId(List<Integer> tocId);
/**
* Traverses the Tree up, until it hits a HeadlineNode or hits a SectionNode which will then return its HeadlineNode.
* Traverses the Tree up, until it hits a HeadlineNode or hits a SectionNode which will then return the first HeadlineNode from its children.
* Throws NotFoundException if no Headline is found this way
*
* @return First HeadlineNode found
*/
@ -69,12 +82,19 @@ public interface SemanticNode {
}
/**
* @return boolean indicating wether this Node has a Parent in the TableOfContents
*/
default boolean hasParent() {
return getTableOfContents().hasParentById(getTocId());
}
/**
* @return The SemanticNode representing the Parent in the TableOfContents
* throws NotFoundException, when no parent is present
*/
default SemanticNode getParent() {
return getTableOfContents().getParentEntryById(getTocId()).node();
@ -83,7 +103,8 @@ public interface SemanticNode {
/**
* Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden.
* Currently only Sections and Tables are not terminal.
* Currently only Sections, Images, and Tables are not terminal.
* A TableCell might be Terminal depending on its area compared to the page.
*
* @return boolean, indicating if a Node has direct access to a TextBlock
*/
@ -122,12 +143,31 @@ public interface SemanticNode {
}
/**
* @return true, if this node's TextBlock is not empty
*/
default boolean hasText() {
return buildTextBlock().length() > 0;
}
/**
* @param string A String
* @return true, if this node's TextBlock contains the string
*/
default boolean containsString(String string) {
return buildTextBlock().getSearchText().contains(string);
}
/**
* THis function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity.
* It sets the fields accordingly and recursively calls this function on all its children.
*
* @param entity EntityNode, which is being inserted into the graph
*/
default void addThisToEntityIfIntersects(EntityNode entity) {
TextBlock textBlock = buildTextBlock();
@ -143,18 +183,42 @@ public interface SemanticNode {
}
/**
* Streams all children located directly underneath this node in the TableOfContents
*
* @return Stream of all children
*/
default Stream<SemanticNode> streamChildren() {
return getTableOfContents().streamChildren(getTocId());
}
/**
* recursively streams all SemanticNodes located underneath this node in the TableOfContents in order.
*
* @return Stream of all SubNodes
*/
default Stream<SemanticNode> streamAllSubNodes() {
return getTableOfContents().streamSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::node);
}
/**
* @return Boundary of this Node's TextBlock
*/
default Boundary getBoundary() {
return buildTextBlock().getBoundary();
}
/**
* If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calcuate the Union of the BoundingBoxes of all Children
*
* @return Rectangle2D fully encapsulating this Node for each page.
*/
default Map<PageNode, Rectangle2D> getBBox() {
Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
@ -166,12 +230,10 @@ public interface SemanticNode {
}
default Boundary getBoundary() {
return buildTextBlock().getBoundary();
}
/**
* @param bBoxPerPage initial empty BoundingBox
* @return The union of the BoundingBoxes of all children
*/
private Map<PageNode, Rectangle2D> getBBoxFromChildren(Map<PageNode, Rectangle2D> bBoxPerPage) {
return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
@ -181,6 +243,10 @@ public interface SemanticNode {
}
/**
* @param bBoxPerPage initial empty BoundingBox
* @return The union of all BoundingBoxes of the TextBlock of this node
*/
private Map<PageNode, Rectangle2D> getBBoxFromTerminalTextBlock(Map<PageNode, Rectangle2D> bBoxPerPage) {
Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage));

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java
View File

@ -8,7 +8,6 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@ -25,7 +24,7 @@ import lombok.experimental.FieldDefaults;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class TableCellNode extends PageElement implements SemanticNode {
public class TableCellNode implements SemanticNode {
List<Integer> tocId;
int row;
@ -60,6 +59,10 @@ public class TableCellNode extends PageElement implements SemanticNode {
@Override
public TextBlock buildTextBlock() {
if (terminal) {
return terminalTextBlock;
}
if (textBlock == null) {
textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
}
@ -76,7 +79,7 @@ public class TableCellNode extends PageElement implements SemanticNode {
public boolean hasHeader(String headerString) {
return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().contains(headerString));
return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().strip().equals(headerString));
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java
View File

@ -5,7 +5,6 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults;
@Builder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class TableNode extends PageElement implements SemanticNode {
public class TableNode implements SemanticNode {
List<Integer> tocId;
TableOfContents tableOfContents;

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
import static java.lang.String.format;
import java.awt.geom.Rectangle2D;
import java.util.LinkedList;
import java.util.List;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
@ -121,11 +120,7 @@ public class AtomicTextBlock implements TextBlock {
.map(RectangleTransformations::rectangleUnion)
.toList();
List<EntityPosition> entityPositions = new LinkedList<>();
for (Rectangle2D position : positionsPerLine) {
entityPositions.add(EntityPosition.builder().position(position).pageNode(page).build());
}
return entityPositions;
return List.of(EntityPosition.builder().rectanglePerLine(positionsPerLine).pageNode(page).build());
}

25
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java
View File

@ -1,14 +1,18 @@
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
import static java.lang.String.format;
import static java.util.stream.Collectors.flatMapping;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import java.awt.geom.Rectangle2D;
import java.util.LinkedList;
import java.util.List;
import java.util.function.Supplier;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import lombok.AccessLevel;
import lombok.Data;
@ -16,7 +20,7 @@ import lombok.experimental.FieldDefaults;
@Data
@FieldDefaults(level = AccessLevel.PRIVATE)
public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTextBlock> {
public class ConcatenatedTextBlock implements TextBlock {
List<AtomicTextBlock> atomicTextBlocks;
String searchText;
@ -68,7 +72,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
public String getSearchText() {
if (searchText == null) {
StringBuilder sb = new StringBuilder();
StringBuilder sb = new StringBuilder();
getAtomicTextBlocks().forEach(atb -> sb.append(atb.getSearchText()));
searchText = sb.toString();
}
@ -96,6 +100,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
return getAtomicTextBlockByStringIndex(fromIndex).getPreviousLinebreak(fromIndex);
}
@Override
public List<Integer> getLineBreaks() {
@ -152,14 +157,20 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
positions.addAll(lastTextBlock.getEntityPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
return positions;
return mergeEntityPositionsWithSamePageNode(positions);
}
@Override
public ConcatenatedTextBlock get() {
private List<EntityPosition> mergeEntityPositionsWithSamePageNode(List<EntityPosition> positions) {
Map<PageNode, List<Rectangle2D>> entityPositionsPerPage = positions.stream().collect(//
groupingBy(EntityPosition::getPageNode, //
flatMapping(entityPosition -> entityPosition.getRectanglePerLine().stream(), toList())));
return entityPositionsPerPage.entrySet().stream()//
.map(entry -> EntityPosition.builder().pageNode(entry.getKey()).rectanglePerLine(entry.getValue()).build())//
.toList();
return this;
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java
View File

@ -99,7 +99,7 @@ public interface TextBlock extends CharSequence {
@Override
default int length() {
return getSearchText().length();
return getBoundary().length();
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java
View File

@ -13,11 +13,10 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class TextBlockCollector implements Collector<TextBlock, ConcatenatedTextBlock, TextBlock> {
@Override
public Supplier<ConcatenatedTextBlock> supplier() {
return new ConcatenatedTextBlock(Collections.emptyList());
return () -> new ConcatenatedTextBlock(Collections.emptyList());
}

104
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java
View File

@ -1,7 +1,8 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@ -9,7 +10,6 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
@ -30,64 +30,79 @@ public class EntityCreationService {
private final EntityTextEnrichmentService entityEnrichmentService;
public Set<EntityNode> createEntitiesByLineAfterString(String string, SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) {
public Set<EntityNode> createEntitiesByBetweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
return Collections.emptySet();
}
public Set<EntityNode> createEntitiesBySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
.stream()
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary))
.map(bounds -> createEntityByBoundary(bounds, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
}
public Set<EntityNode> createEntitiesByLineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
SearchImplementation searchImplementation = new SearchImplementation(List.of(string), true);
List<Boundary> boundaries = searchImplementation.getBoundaries(textBlock, node.getBoundary());
return boundaries.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
.map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph))
.map(boundary -> createEntityByBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
}
public Set<EntityNode> createEntitiesByRegex(String regexPattern, String type, EntityType entityType, DocumentGraph documentGraph) {
public Set<EntityNode> createEntitiesByRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
List<Boundary> boundaries = RegexMatcher.findBoundaries(regexPattern, documentGraph.buildTextBlock());
return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph)).collect(Collectors.toUnmodifiableSet());
List<Boundary> boundaries = RegexMatcher.findBoundaries(regexPattern, node.buildTextBlock());
return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
}
public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, DocumentGraph documentGraph) {
public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType) {
Boundary boundary = node.buildTextBlock().getBoundary();
Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node);
}
public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType);
addEntityToGraph(entity, documentGraph);
addEntityToGraph(entity, node.getTableOfContents());
return entity;
}
public EntityNode createEntityByEntity(EntityNode entity, String type, EntityType entityType, DocumentGraph documentGraph) {
public EntityNode createMergedEntity(List<EntityNode> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
return createEntityByBoundary(entity.getBoundary(), type, entityType, documentGraph);
}
public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) {
Boundary boundary = node.buildTextBlock().getBoundary();
Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, documentGraph);
}
public Set<EntityNode> createEntitiesByString(String entityName, int startOffset, String type, EntityType entityType, DocumentGraph documentGraph) {
int start = documentGraph.buildTextBlock().indexOf(entityName, startOffset);
if (start == -1) {
throw new NotFoundException(format("Entity %s could no be found!", entityName));
if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
throw new IllegalArgumentException("Provided entities can not be merged!");
}
EntityNode entity = EntityNode.initialEntityNode(new Boundary(start, start + entityName.length()), type, entityType);
addEntityToGraph(entity, documentGraph);
return Set.of(entity);
EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType);
mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction));
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
if (mergedEntity.isRedaction()) {
mergedEntity.setRedactionReason(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getRedactionReason).findFirst().orElse(""));
mergedEntity.setMatchedRule(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getMatchedRule).findFirst().orElse(-1));
mergedEntity.setLegalBasis(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getLegalBasis).findFirst().orElse(""));
}
addEntityToGraph(mergedEntity, node.getTableOfContents());
return mergedEntity;
}
public void addEntityToGraph(EntityNode entity, DocumentGraph documentGraph) {
public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
try {
SemanticNode containingNode = documentGraph.getTableOfContents()
.getEntries()
SemanticNode containingNode = tableOfContents.getEntries()
.stream()
.map(TableOfContents.Entry::node)
.filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
@ -103,7 +118,7 @@ public class EntityCreationService {
addToNodeEntitySets(entity);
} catch (NotFoundException e) {
entityEnrichmentService.enrichEntity(entity, documentGraph.getTextBlock());
entityEnrichmentService.enrichEntity(entity, tableOfContents.buildTextBlock());
log.warn("Entity \"{}\" with {} is in between two main sections and will be removed!", entity.getValue(), entity.getBoundary());
entity.removeFromGraph();
}
@ -123,6 +138,27 @@ public class EntityCreationService {
entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
}
private static boolean allEntitiesIntersectAndHaveSameTypes(List<EntityNode> entitiesToMerge) {
if (entitiesToMerge.isEmpty()) {
return true;
}
EntityNode previousEntity = entitiesToMerge.get(0);
for (EntityNode entityNode : entitiesToMerge.subList(1, entitiesToMerge.size())) {
boolean typeMatches = entityNode.getType().equals(previousEntity.getType());
boolean entityTypeMatches = entityNode.getEntityType().equals(previousEntity.getEntityType());
boolean intersects = entityNode.intersects(previousEntity);
if (typeMatches && entityTypeMatches && intersects) {
previousEntity = entityNode;
continue;
}
return false;
}
return true;
}
private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) {
return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end()));

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java Normal file
View File

@ -0,0 +1,32 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import java.util.Collection;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
public class EntityFieldSetter {
public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
if (entityNode.getMatchedRule() == -1 && matchedRule != -1) {
entityNode.setMatchedRule(matchedRule);
}
if (entityNode.getRedactionReason().equals("") && redactionReason != null) {
entityNode.setRedactionReason(redactionReason);
}
if (entityNode.getLegalBasis().equals("") && legalBasis != null) {
entityNode.setLegalBasis(legalBasis);
}
if (engine != null) {
entityNode.addEngine(engine);
}
}
public static void setFields(Collection<EntityNode> entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
}
}

23
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java
View File

@ -1,19 +1,29 @@
package com.iqser.red.service.redaction.v1.server.document.services;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
public class RegexMatcher {
public static boolean anyMatch(CharSequence searchText, String regexPattern) {
public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
return pattern.matcher(searchText).find();
return pattern.matcher(charSequence).find();
}
public static boolean anyMatch(TextBlock textBlock, String regexPattern) {
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
return pattern.matcher(textBlock).region(textBlock.getBoundary().start(), textBlock.getBoundary().end()).find();
}
@ -24,15 +34,16 @@ public class RegexMatcher {
return new Boundary(matcher.start(), matcher.end());
}
public static List<Boundary> findBoundaries(String regexPattern, CharSequence searchText) {
public static List<Boundary> findBoundaries(String regexPattern, TextBlock textBlock) {
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
Matcher matcher = pattern.matcher(searchText);
Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary()));
List<Boundary> boundaries = new LinkedList<>();
while (matcher.find()) {
boundaries.add(new Boundary(matcher.start(), matcher.end()));
boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start()));
}
return boundaries;
return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
}
}

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java
View File

@ -1,11 +1,16 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import static java.lang.String.format;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import lombok.Data;
import lombok.Getter;
@ -50,8 +55,11 @@ public class Dictionary {
public DictionaryModel getType(String type) {
DictionaryModel model = localAccessMap.get(type);
if (model == null) throw new NotFoundException("Type: " + type + " is not found");
if (model == null) {
throw new NotFoundException("Type: " + type + " is not found");
}
return model;
}
@ -75,4 +83,23 @@ public class Dictionary {
return false;
}
public void addLocalDictionaryEntry(String type, String value, boolean alsoAddLastname) {
if (localAccessMap.get(type) == null) {
throw new IllegalArgumentException(format("DictionaryModel of type %s does not exist", type));
}
if (localAccessMap.get(type).getLocalEntries() == null) {
throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type));
}
if (StringUtils.isEmpty(value) && value.length() < 3) {
throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
}
localAccessMap.get(type).getLocalEntries().add(value);
if (alsoAddLastname) {
String lastname = value.split(" ")[0];
localAccessMap.get(type).getLocalEntries().add(lastname);
}
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -104,10 +104,12 @@ public class DroolsExecutionService {
}
@Timed("redactmanager_executeRules")
public Section executeRules(KieContainer kieContainer, Section section) {
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("section", section);
kieSession.insert(section);
kieSession.fireAllRules();
kieSession.dispose();
@ -116,7 +118,9 @@ public class DroolsExecutionService {
}
public List<Entity> getEntities(KieSession ks) {
List<Entity> entities = new LinkedList<>();
QueryResults entitiesResult = ks.getQueryResults("getEntities");
for (QueryResultsRow resultsRow : entitiesResult) {

69
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -1,7 +1,8 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -14,7 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@ -35,19 +36,73 @@ public class RedactionLogCreatorService {
private final DictionaryService dictionaryService;
public List<RedactionLogEntry> createRedactionLog(Map<PageNode, Set<EntityNode>> entityNodesPerPageNode, int numberOfPages, String dossierTemplateId) {
public List<RedactionLogEntry> createRedactionLog(Set<EntityNode> entityNodes, String dossierTemplateId) {
List<RedactionLogEntry> entries = new ArrayList<>();
entityNodesPerPageNode.forEach(((pageNode, entityNodes) -> entries.addAll(toRedactionLogEntries(pageNode, entityNodes, dossierTemplateId))));
Set<String> processedIds = new HashSet<>();
entityNodes.forEach((entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId))));
return entries;
}
private Set<RedactionLogEntry> toRedactionLogEntries(PageNode pageNode, Set<EntityNode> entityNodes, String dossierTemplateId) {
private List<RedactionLogEntry> toRedactionLogEntries(EntityNode entityNode, Set<String> processedIds, String dossierTemplateId) {
return Collections.emptySet();
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
// Duplicates can exist due table extraction columns over multiple rows.
for (EntityPosition entityPosition : entityNode.getEntityPositions()) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entityNode, dossierTemplateId);
if (processedIds.contains(entityPosition.getId())) {
continue;
}
processedIds.add(entityPosition.getId());
redactionLogEntry.setId(entityPosition.getId());
List<Rectangle> rectanglesPerLine = entityPosition.getRectanglePerLine()
.stream()
.map(rectangle2D -> toRectangle(rectangle2D, entityPosition.getPageNode().getNumber()))
.toList();
redactionLogEntry.setPositions(rectanglesPerLine);
}
return redactionLogEntities;
}
private RedactionLogEntry createRedactionLogEntry(EntityNode entity, String dossierTemplateId) {
Set<String> referenceIds = new HashSet<>();
entity.getReferences().forEach(ref -> ref.getEntityPositions().forEach(pos -> referenceIds.add(pos.getId())));
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
.reason(entity.getRedactionReason())
.legalBasis(entity.getLegalBasis())
.value(entity.getValue())
.type(entity.getType())
.redacted(entity.isRedaction())
.isHint(isHint(entity.getType(), dossierTemplateId))
.isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(entity.getDeepestFullyContainingNode().getTocId().toString())
.sectionNumber(entity.getDeepestFullyContainingNode().getTocId().get(0))
.matchedRule(entity.getMatchedRule())
.isDictionaryEntry(entity.isDictionaryEntry())
.textAfter(entity.getTextAfter().toString())
.textBefore(entity.getTextBefore().toString())
.startOffset(entity.getBoundary().start())
.endOffset(entity.getBoundary().end())
.isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
.engines(entity.getEngines())
.reference(referenceIds)
.build();
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -124,7 +124,7 @@ public final class EntitySearchUtils {
orderedEntities.get(i).setPositionSequences(toAdd);
} catch (Exception e) {
log.warn("Mismatch between EntityPositionSequence and found Entity!");
//log.warn("Mismatch between EntityPositionSequence and found Entity!");
}
}
}
@ -209,9 +209,7 @@ public final class EntitySearchUtils {
.length() && inner.getStart() >= outer.getStart() && inner.getEnd() <= outer.getEnd() && outer != inner && outer.getSectionNumber() == inner.getSectionNumber()) {
if (outer.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
wordsToRemove.add(outer);
} else if (!(
inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && outer.getEntityType() == EntityType.ENTITY ||
inner.getEntityType() == EntityType.ENTITY && outer.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
} else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && outer.getEntityType() == EntityType.ENTITY || inner.getEntityType() == EntityType.ENTITY && outer.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
if (inner.isResized()) {
wordsToRemove.add(outer);
} else {

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java
View File

@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
import java.util.Set;
import java.util.regex.Pattern;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@ -28,4 +30,23 @@ public final class SeparatorUtils {
return intValue >= 12288 && intValue <= 12336 || japaneseAltPunctuationMarks.contains(intValue);
}
public static boolean validateBoundaryIsSurroundedBySeparators(CharSequence charSequence, Boundary boundary) {
return validateStart(charSequence, boundary) && validateEnd(charSequence, boundary);
}
private static boolean validateEnd(CharSequence charSequence, Boundary boundary) {
return boundary.end() == charSequence.length() || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end())) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end() - 1));
}
private static boolean validateStart(CharSequence charSequence, Boundary boundary) {
return boundary.start() == 0 || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start()));
}
}

53
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
import static com.iqser.red.service.redaction.v1.server.utils.PdfDraw.drawRectangle2DList;
import java.awt.Color;
@ -18,10 +19,9 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
@ -29,11 +29,11 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.utils.PdfDraw;
@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
@SneakyThrows
public void testDroolsOnDocumentGraph() {
String filename = "files/new/crafted document";
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
prepareStorage(filename + ".pdf");
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
@ -76,31 +76,39 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
long dictionarySearchStart = System.currentTimeMillis();
List<EntityNode> foundEntities = new LinkedList<>();
for (var model : dictionary.getDictionaryModels()) {
for (DictionaryModel model : dictionary.getDictionaryModels()) {
findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document));
}
System.out.printf("Dictionary search took %d ms and found %d entities\n", System.currentTimeMillis() - dictionarySearchStart, foundEntities.size());
long graphInsertionStart = System.currentTimeMillis();
foundEntities.forEach(entity -> entity.setDossierDictionaryEntry(true));
foundEntities.forEach(entity -> entity.setDictionaryEntry(true));
foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document.getTableOfContents()));
System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart);
KieSession kieSession = kieContainer.newKieSession();
entityCreationService.createEntitiesByString("Michael N.", 0, "CBI_author", EntityType.FALSE_POSITIVE, document)//
.forEach(kieSession::insert);
kieSession.setGlobal("document", document);
kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("dictionary", dictionary);
long serializationStart = System.currentTimeMillis();
document.getEntities().forEach(kieSession::insert);
document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node()));
document.getPages().forEach(kieSession::insert);
System.out.printf("Object serialization and kieSession insertion took %d ms\n", System.currentTimeMillis() - serializationStart);
long start = System.currentTimeMillis();
long dictionaryAddsStart = System.currentTimeMillis();
kieSession.insert(FileAttribute.builder().label("Vertebrate Study").value("Yes").build());
kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
kieSession.fireAllRules();
System.out.printf("Firing all rules took %d ms\n", System.currentTimeMillis() - start);
//List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog();
System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - dictionaryAddsStart);
System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart);
List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document.getEntities(), TEST_DOSSIER_TEMPLATE_ID);
drawAllEntities(filename, fileResource, document);
}
@ -181,7 +189,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
var insertStart = System.currentTimeMillis();
DocumentGraph finalDocument = document;
foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument));
foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument.getTableOfContents()));
var insertTime = System.currentTimeMillis() - insertStart;
totalInsertTime += insertTime;
@ -210,7 +218,8 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
.filter(entityNode -> !entityNode.isRemoved())
.filter(EntityNode::isRedaction)
.flatMap(entityNode -> entityNode.getEntityPositions().stream())
.map(EntityPosition::getPosition)
.filter(entityPosition -> entityPosition.getPageNode().equals(page))
.flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
.toList();
PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLACK).stroke(true).build();
@ -221,9 +230,10 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
List<Rectangle2D> entityPositionsOnPage = page.getEntities()
.stream()
.filter(entityNode -> !entityNode.isRemoved())
.filter(e -> !e.isRedaction())
.filter(entityNode -> !entityNode.isRedaction())
.flatMap(entityNode -> entityNode.getEntityPositions().stream())
.map(EntityPosition::getPosition)
.filter(entityPosition -> entityPosition.getPageNode().equals(page))
.flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
.toList();
PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLUE).stroke(true).build();
@ -245,16 +255,9 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
TextBlock textBlock = documentGraph.getTextBlock();
searchImplementation.getBoundaries(textBlock, textBlock.getBoundary())
.stream()
.filter(boundary -> validateSeparators(textBlock, boundary))
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
.map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType))
.forEach(foundEntities::add);
}
private static boolean validateSeparators(TextBlock textBlock, Boundary boundary) {
return (boundary.start() == 0 || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start()))) && (boundary.end() == textBlock.length() || SeparatorUtils.isSeparator(
textBlock.charAt(boundary.end())) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1)));
}
}

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java
View File

@ -7,8 +7,8 @@ import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
@ -46,11 +46,6 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
assert document.toString().equals(newDocumentGraph.toString());
assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString());
for (int pageIndex = 1; pageIndex < document.getNumberOfPages(); pageIndex++) {
var page = document.getPages().get(pageIndex);
var newPage = document.getPages().get(pageIndex);
assert page.toString().equals(newPage.toString());
}
}
}

20
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java
View File

@ -14,16 +14,16 @@ import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@ -56,7 +56,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("Expand to Hint ", entityNode.getTextBefore());
assertEquals("’s Donut ←", entityNode.getTextAfter());
@ -80,7 +80,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("", entityNode.getTextBefore());
assertEquals(" Purity Hint", entityNode.getTextAfter());
@ -103,7 +103,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("", entityNode.getTextBefore());
assertEquals("", entityNode.getTextAfter());
@ -189,7 +189,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore());
assertEquals(", Group 9;", entityNode.getTextAfter());
@ -238,7 +238,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore());
assertEquals(" of metabolite of", entityNode.getTextAfter());
@ -293,8 +293,8 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
PageNode pageNode = documentGraph.getPages().get(pageNumber - 1);
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
assertEquals(entityNode.getValue(), searchTerm);
assertTrue(pageNode.getEntities().contains(entityNode));

109
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl
View File

@ -2,6 +2,7 @@ package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields;
import java.util.List;
@ -17,9 +18,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
global DocumentGraph document
global EntityCreationService entityCreationService
global Dictionary dictionary
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "1: Redact CBI Authors (Non vertebrate study)"
no-loop true
@ -29,14 +34,7 @@ rule "1: Redact CBI Authors (Non vertebrate study)"
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
if ($entity.getMatchedRule() == -1) {
$entity.setMatchedRule(1);
}
if ($entity.getRedactionReason().equals("")) {
$entity.setRedactionReason("Author found");
}
$entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.addEngine(Engine.RULE);
setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
@ -48,14 +46,7 @@ rule "2: Redact CBI Authors (Vertebrate study)"
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
if ($entity.getMatchedRule() == -1) {
$entity.setMatchedRule(2);
}
if ($entity.getRedactionReason().equals("")) {
$entity.setRedactionReason("Author found");
}
$entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.addEngine(Engine.RULE);
setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
@ -65,15 +56,9 @@ rule "3: Don't redact CBI Address (Non vertebrate study)"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
$recommendationEntity: EntityNode(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
then
$entity.setRedaction(false);
$entity.setMatchedRule(3);
$entity.setRedactionReason("Address found for non vertebrate study");
$entity.addEngine(Engine.RULE);
setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
$recommendationEntity.removeFromGraph();
delete($recommendationEntity);
end
rule "4: Redact CBI Address (Vertebrate study)"
@ -84,43 +69,43 @@ rule "4: Redact CBI Address (Vertebrate study)"
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
$entity.setMatchedRule(4);
$entity.setRedactionReason("Address found");
$entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.addEngine(Engine.RULE);
setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
when
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
then
EntityNode entity = entityCreationService.createEntityByEntity($entity, "CBI_author", EntityType.FALSE_POSITIVE, document);
entity.setMatchedRule(5);
EntityNode entity = entityCreationService.createEntityByBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
setFields($entity, 5, "Genitive Author", null, Engine.RULE);
insert(entity);
end
rule "6: Create Entity from Author(s) cells in Tables with Author(s) header"
rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
authorCell: TableCellNode(header == false, (hasHeader("Author(s)") || hasHeader("Author")))
authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
then
EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY, document);
entity.setMatchedRule(6);
entity.setRedactionReason("Header \"Author(s)\" found");
EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
end
rule "7: Add CBI_author with \"et al.\" Regex"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: SectionNode(containsString("et al."))
then
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, document);
entities.forEach(entity -> entity.setMatchedRule(7));
entities.forEach(entity -> entity.setRedactionReason("Found by \"et al.\" regex"));
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
end
rule "8: Add recommendation for Addresses in Test Organism sections"
@ -129,9 +114,8 @@ rule "8: Add recommendation for Addresses in Test Organism sections"
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species") && containsString("Source"))
then
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source", $section, "CBI_address", EntityType.RECOMMENDATION, document);
entities.forEach(entity -> entity.setRedactionReason("Line after \"Source\" in Test Organism Section"));
entities.forEach(entity -> entity.setMatchedRule(8));
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
@ -142,9 +126,46 @@ rule "9: Add recommendation for Addresses in Test Animals sections"
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species:") && containsString("Source:"))
then
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source:", $section, "CBI_address", EntityType.RECOMMENDATION, document);
entities.forEach(entity -> entity.setRedactionReason("Line after \"Source:\" in Test Organism Section"));
entities.forEach(entity -> entity.setMatchedRule(9));
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
rule "12: Redact Emails by RegEx (Non vertebrate study)"
when
$section: SectionNode(containsString("@"))
then
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 12, "Found by email regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end

83
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl
View File

@ -8,75 +8,88 @@ import java.util.List;
import java.util.LinkedList;
import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.document.graph.*
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
global DocumentGraph document
global EntityCreationService entityCreationService
global Dictionary dictionary
rule "merge contained Entities of same type"
rule "merge intersecting Entities of same type"
salience 100
when
outer: EntityNode($type: type, $entityType: entityType)
inner: EntityNode(this != outer, containedBy(outer), type == $type, entityType == $entityType)
$first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
then
System.out.printf("removed entity %s contained by %s\n", inner, outer);
outer.addEngines(inner.getEngines());
inner.removeFromGraph();
delete(inner);
$first.removeFromGraph();
$second.removeFromGraph();
EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document);
insert(mergedEntity);
delete($first);
delete($second);
end
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 100
when
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
entity: EntityNode(this != $falsePositive, containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY)
entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then
System.out.printf("removed entity %s marked false positive by %s\n", entity, $falsePositive);
entity.removeFromGraph();
delete(entity);
end
/*
rule "If same type, remove inner"
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 100
when
$containedEntity: ContainedEntity($toRemove: inner, outer.getType() == inner.getType())
$falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$toRemove.removeFromGraph();
delete($toRemove);
delete($containedEntity);
$recommendation.removeFromGraph();
delete($recommendation);
end
rule "If outer Recommended and inner Entity, remove outer"
salience 99
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 100
when
$containedEntity: ContainedEntity($toRemove: outer, outer.getType() == EntityType.RECOMMENDATION, inner.getType() == EntityType.ENTITY)
$entity: EntityNode($type: type, entityType == EntityType.ENTITY)
$recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$toRemove.removeFromGraph();
delete($toRemove);
delete($containedEntity);
$recommendation.removeFromGraph();
delete($recommendation);
end
rule "Inner not False Recommendation and outer not Entity"
salience 99
rule "remove Entity of lower rank, when equal boundaries and entityType"
salience 100
when
$containedEntity: ContainedEntity($toRemove: outer, (outer.getType() != EntityType.ENTITY || inner.getType() != EntityType.FALSE_RECOMMENDATION))
$higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
$lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
then
$toRemove.removeFromGraph();
delete($toRemove);
delete($containedEntity);
end
rule "Remove leftover ContainedEntities"
salience 0
when
$toRemove: ContainedEntity()
then
delete($toRemove);
$lowerRank.removeFromGraph();
delete($lowerRank);
end
*/
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
Set<EntityNode> entityNodes = entityCreationService.createEntitiesBySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
entityNodes.forEach(entityNode -> insert(entityNode));
end

367
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -0,0 +1,367 @@
package drools
import com.iqser.red.service.redaction.v1.server.redaction.model.Section
global Section section
// --------------------------------------- AI rules -------------------------------------------------------------------
rule "0: Add CBI_author from ai"
when
Section(aiMatchesType("CBI_author"))
then
section.addAiEntities("CBI_author", "CBI_author");
end
rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
when
Section(aiMatchesType("ORG"))
then
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
end
rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
when
Section(aiMatchesType("STREET"))
then
section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
end
rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
when
Section(aiMatchesType("CITY"))
then
section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "1: Redact CBI Authors (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
then
section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "2: Redact CBI Authors (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
then
section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "3: Redact not CBI Address (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
then
section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
section.ignoreRecommendations("CBI_address");
end
rule "4: Redact CBI Address (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
then
section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "5: Do not redact genitive CBI_author"
when
Section(matchesType("CBI_author"))
then
section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
end
rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
then
section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
then
section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
then
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "16: Add recommendation for Addresses in Test Organism sections"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
then
section.recommendLineAfter("Source:", "CBI_address");
end
rule "17: Add recommendation for Addresses in Test Animals sections"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
then
section.recommendLineAfter("Source", "CBI_address");
end
rule "18: Do not redact Names and Addresses if Published Information found"
when
Section(matchesType("published_information"))
then
section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
then
section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
then
section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "21: Redact Emails by RegEx (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "22: Redact Emails by RegEx (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
then
section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "23: Redact contact information (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|| text.contains("Contact:")
|| text.contains("Alternative contact:")
|| (text.contains("No:") && text.contains("Fax"))
|| (text.contains("Contact:") && text.contains("Tel.:"))
|| text.contains("European contact:")
))
then
section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "24: Redact contact information (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
|| text.contains("Contact:")
|| text.contains("Alternative contact:")
|| (text.contains("No:") && text.contains("Fax"))
|| (text.contains("Contact:") && text.contains("Tel.:"))
|| text.contains("European contact:")
))
then
section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Contact")
|| text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
text.contains("Contact")
|| text.contains("Telephone")
|| text.contains("Phone")
|| text.contains("Fax")
|| text.contains("Tel")
|| text.contains("Ter")
|| text.contains("Mobile")
|| text.contains("Fel")
|| text.contains("Fer")
))
then
section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "27: Redact AUTHOR(S) (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "28: Redact AUTHOR(S) (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("COMPLETION DATE:")
&& !searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "29: Redact AUTHOR(S) (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "30: Redact AUTHOR(S) (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("AUTHOR(S):")
&& searchText.contains("STUDY COMPLETION DATE:")
)
then
section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:")
)
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
end
rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
&& searchText.contains("PERFORMING LABORATORY:"))
then
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "33: Redact study director abbreviation"
when
Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
then
section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "34: Purity Hint"
when
Section(searchText.toLowerCase().contains("purity"))
then
section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
end
rule "35: Redact signatures (Non vertebrate study)"
when
Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "36: Redact signatures (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
then
section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "43: Redact Logos (Vertebrate study)"
when
Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
then
section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end