From b6307759fba635944f4d95a921c5038df715652a Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 22 Mar 2023 19:05:33 +0100 Subject: [PATCH] RED-6369: Rules Refactor *added some new Rules and Methods to support them *integrated local dictionary searches into ruleflow using agenda-groups *Generalized methods to be able to execute on any SemanticNode *made Documentgraph implement SemanticNode --- .../data/mapper/DocumentDataMapper.java | 2 +- .../data/mapper/DocumentGraphMapper.java | 10 +- .../v1/server/document/graph/Boundary.java | 13 +- .../server/document/graph/DocumentGraph.java | 41 +- .../v1/server/document/graph/PageElement.java | 14 - .../document/graph/TableOfContents.java | 13 +- .../document/graph/entity/EntityNode.java | 16 +- .../document/graph/entity/EntityPosition.java | 9 +- .../document/graph/entity/ImageNode.java | 3 +- .../graph/factory/DocumentGraphFactory.java | 47 ++- .../factory/RectangleTransformations.java | 8 + .../SearchTextWithTextPositionFactory.java | 8 +- .../graph/factory/TextBlockFactory.java | 16 +- .../document/graph/nodes/FooterNode.java | 3 +- .../document/graph/nodes/HeaderNode.java | 4 +- .../document/graph/nodes/HeadlineNode.java | 3 +- .../document/graph/nodes/ParagraphNode.java | 5 +- .../document/graph/nodes/SectionNode.java | 3 +- .../document/graph/nodes/SemanticNode.java | 96 ++++- .../document/graph/nodes/TableCellNode.java | 9 +- .../document/graph/nodes/TableNode.java | 3 +- .../graph/textblock/AtomicTextBlock.java | 7 +- .../textblock/ConcatenatedTextBlock.java | 25 +- .../document/graph/textblock/TextBlock.java | 2 +- .../graph/textblock/TextBlockCollector.java | 3 +- .../services/EntityCreationService.java | 104 +++-- .../document/services/EntityFieldSetter.java | 32 ++ .../document/services/RegexMatcher.java | 23 +- .../v1/server/redaction/model/Dictionary.java | 29 +- .../service/DroolsExecutionService.java | 4 + .../service/RedactionLogCreatorService.java | 69 +++- .../redaction/utils/EntitySearchUtils.java | 6 +- .../redaction/utils/SeparatorUtils.java | 21 + .../server/DocumentGraphIntegrationTest.java | 53 +-- .../graph/DocumentGraphMappingTest.java | 7 +- .../document/graph/DocumentGraphTest.java | 20 +- .../test/resources/drools/entity_rules.drl | 109 +++--- .../resources/drools/merge_entity_rules.drl | 83 ++-- .../src/test/resources/drools/rules.drl | 367 ++++++++++++++++++ 39 files changed, 1011 insertions(+), 279 deletions(-) delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageElement.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java index 66755991..d1a98092 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java @@ -20,7 +20,7 @@ public class DocumentDataMapper { public DocumentData toDocumentData(DocumentGraph documentGraph) { - List atomicTextBlockData = documentGraph.streamTextBlocksInOrder() + List atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder() .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) .distinct() .map(this::toAtomicTextBlockData) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java index 209387d1..4e607630 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java @@ -22,12 +22,12 @@ import com.iqser.red.service.redaction.v1.server.document.data.PageData; import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; @@ -44,7 +44,11 @@ public class DocumentGraphMapper { buildNodesFromTableOfContents(Collections.emptyList(), context); - DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(documentData.getPages().size()).pages(context.pages).tableOfContents(context.tableOfContents).build(); + DocumentGraph documentGraph = DocumentGraph.builder() + .numberOfPages(documentData.getPages().size()) + .pages(new HashSet<>(context.pages)) + .tableOfContents(context.tableOfContents) + .build(); documentGraph.setTextBlock(documentGraph.buildTextBlock()); return documentGraph; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java index 56b9ed70..8f028036 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java @@ -86,7 +86,7 @@ public class Boundary implements Comparable { public List split(List splitIndices) { - if(splitIndices.stream().anyMatch(idx -> !this.contains(idx))) { + if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) { throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this)); } List splitBoundaries = new LinkedList<>(); @@ -100,6 +100,14 @@ public class Boundary implements Comparable { } + public static Boundary merge(List boundaries) { + + int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new); + int maxEnd = boundaries.stream().mapToInt(Boundary::end).max().orElseThrow(IllegalArgumentException::new); + return new Boundary(minStart, maxEnd); + } + + @Override public String toString() { @@ -109,6 +117,7 @@ public class Boundary implements Comparable { @Override public int compareTo(Boundary boundary) { + if (end < boundary.end() && start < boundary.start()) { return -1; } @@ -119,12 +128,14 @@ public class Boundary implements Comparable { return 0; } + @Override public int hashCode() { return toString().hashCode(); } + @Override public boolean equals(Object object) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java index e386673b..580255a8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java @@ -1,14 +1,18 @@ package com.iqser.red.service.redaction.v1.server.document.graph; +import java.awt.geom.Rectangle2D; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; @@ -24,9 +28,9 @@ import lombok.extern.slf4j.Slf4j; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class DocumentGraph { +public class DocumentGraph implements SemanticNode { - List pages; + Set pages; TableOfContents tableOfContents; Integer numberOfPages; TextBlock textBlock; @@ -34,7 +38,7 @@ public class DocumentGraph { public TextBlock buildTextBlock() { - return streamTextBlocksInOrder().collect(new TextBlockCollector()); + return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector()); } @@ -44,7 +48,7 @@ public class DocumentGraph { } - public Stream streamTextBlocksInOrder() { + public Stream streamTerminalTextBlocksInOrder() { return streamAllNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock); } @@ -56,6 +60,20 @@ public class DocumentGraph { } + @Override + public List getTocId() { + + return Collections.emptyList(); + } + + + @Override + public void setTocId(List tocId) { + + throw new UnsupportedOperationException("DocumentGraph is always the root of the Table of Contents"); + } + + private Stream streamAllNodes() { return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node); @@ -68,4 +86,15 @@ public class DocumentGraph { return tableOfContents.toString(); } + + @Override + public Map getBBox() { + + Map bBox = new HashMap<>(); + for (PageNode page : pages) { + bBox.put(page, new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight())); + } + return bBox; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageElement.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageElement.java deleted file mode 100644 index ef5ad795..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageElement.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.Map; - -import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; - -import lombok.Data; - -@Data -public class PageElement { - - Map positionsPerPage; - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java index 9913b04e..2bbce104 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java @@ -11,8 +11,11 @@ import java.util.List; import java.util.stream.Stream; import com.google.common.hash.Hashing; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import lombok.Builder; import lombok.Data; @@ -29,6 +32,12 @@ public class TableOfContents { } + public TextBlock buildTextBlock() { + + return streamEntriesInOrder().map(Entry::node).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector()); + } + + public List createNewEntryAndReturnId(NodeType nodeType, SemanticNode node) { return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node); @@ -72,7 +81,7 @@ public class TableOfContents { List parentIds = getParentId(tocId); if (parentIds.size() < 1) { - throw new UnsupportedOperationException(format("Node with tocId \"%s\" has no parent!", tocId)); + throw new NotFoundException(format("Node with tocId \"%s\" has no parent!", tocId)); } return getEntryById(parentIds); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java index d9d06d66..58608056 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java @@ -32,7 +32,6 @@ public class EntityNode { .entityType(entityType) .boundary(boundary) .redaction(false) - .falsePositive(false) .removed(false) .ignored(false) .resized(false) @@ -55,7 +54,6 @@ public class EntityNode { // empty defaults boolean redaction; - boolean falsePositive; boolean removed; boolean ignored; boolean resized; @@ -117,17 +115,30 @@ public class EntityNode { } + public boolean intersects(EntityNode entityNode) { + + return this.boundary.intersects(entityNode.getBoundary()); + } + + public void addEngine(Engine engine) { engines.add(engine); } + public void addEngines(Set engines) { this.engines.addAll(engines); } + public void addReference(EntityNode entityNode) { + + references.add(entityNode); + } + + @Override public String toString() { @@ -157,6 +168,7 @@ public class EntityNode { return Hashing.murmur3_128().hashString(toString(), StandardCharsets.UTF_8).hashCode(); } + @Override public boolean equals(Object o) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java index 29aa1cea..24a1336c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph.entity; import java.awt.geom.Rectangle2D; import java.nio.charset.StandardCharsets; +import java.util.List; import com.google.common.hash.Hashing; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; @@ -17,7 +18,7 @@ import lombok.experimental.FieldDefaults; public class EntityPosition { PageNode pageNode; - Rectangle2D position; + List rectanglePerLine; public String getId() { @@ -29,8 +30,10 @@ public class EntityPosition { @Override public int hashCode() { - String string = String.valueOf(pageNode.getNumber()) + position.getX() + position.getY() + position.getWidth() + position.getHeight(); - return Hashing.murmur3_128().hashString(string, StandardCharsets.UTF_8).hashCode(); + StringBuilder sb = new StringBuilder(); + sb.append(pageNode.getNumber()); + rectanglePerLine.forEach(r -> sb.append(r.getX()).append(r.getY()).append(r.getWidth()).append(r.getHeight())); + return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java index f939eac1..3f620fbe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; @@ -30,7 +29,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class ImageNode extends PageElement implements SemanticNode { +public class ImageNode implements SemanticNode { List tocId; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java index e31ed12a..f22f2f45 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java @@ -10,7 +10,6 @@ import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -28,16 +27,16 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Header; import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; @@ -71,11 +70,7 @@ public class DocumentGraphFactory { addSections(document, context); addHeaderAndFooterToEachPage(document, context); - DocumentGraph documentGraph = DocumentGraph.builder() - .numberOfPages(context.pages.size()) - .pages(context.pages.keySet().stream().sorted(Comparator.comparingInt(PageNode::getNumber)).toList()) - .tableOfContents(context.tableOfContents) - .build(); + DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(context.pages.size()).pages(context.pages.keySet()).tableOfContents(context.tableOfContents).build(); documentGraph.setTextBlock(documentGraph.buildTextBlock()); return documentGraph; @@ -193,28 +188,44 @@ public class DocumentGraphFactory { tableCellNode.setTocId(tocId); if (cell.getTextBlocks().isEmpty()) { - tableCellNode.setTerminal(false); - return; - } + tableCellNode.setTerminalTextBlock(context.textBlockFactory.emptyTextBlock(parentNode, context, page)); + tableCellNode.setTerminal(true); - if (cell.getTextBlocks().size() == 1) { + } else if (cell.getTextBlocks().size() == 1) { textBlock = context.textBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCellNode, context, page); tableCellNode.setTerminalTextBlock(textBlock); - } else if (StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H")) { - tableCellNode.setTerminal(false); + tableCellNode.setTerminal(true); + + } else if (firstTextBlockIsHeadline(cell)) { addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractTextContainer) tb).toList(), Collections.emptyList(), context); - } else if (cell.getArea() < TABLE_CELL_MERGE_SIZE_THRESHOLD * page.getHeight() * page.getWidth()) { + tableCellNode.setTerminal(false); + + } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) { List sequences = mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks()); textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page); tableCellNode.setTerminalTextBlock(textBlock); + tableCellNode.setTerminal(true); + } else { - tableCellNode.setTerminal(false); cell.getTextBlocks().forEach(tb -> addParagraphOrHeadline(tableCellNode, tb, context)); + tableCellNode.setTerminal(false); } } + private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, PageNode page) { + + return cell.getArea() < TABLE_CELL_MERGE_SIZE_THRESHOLD * page.getHeight() * page.getWidth(); + } + + + private static boolean firstTextBlockIsHeadline(Cell cell) { + + return StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H"); + } + + private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context) { addParagraphOrHeadline(parentNode, originalTextBlock, context, Collections.emptyList()); @@ -340,7 +351,7 @@ public class DocumentGraphFactory { PageNode page = getPage(pageIndex, context); HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build(); - AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, context, 0, page); + AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page); List tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header); header.setTocId(tocId); header.setTerminalTextBlock(textBlock); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java index 9353d7ab..77f08036 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java @@ -10,6 +10,8 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collector; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; @@ -52,6 +54,12 @@ public class RectangleTransformations { } + public static Rectangle toRectangle(Rectangle2D rectangle2D, int pageNumber) { + + return new Rectangle(new Point((float) rectangle2D.getMaxX(), (float) rectangle2D.getMaxY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber); + } + + private static class Rectangle2DUnion implements Collector { @Override diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java index 63e02bec..e97e1b11 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java @@ -18,6 +18,7 @@ public class SearchTextWithTextPositionFactory { public static final int HEIGHT_PADDING = 2; + public static SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List sequences) { if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) { @@ -79,8 +80,9 @@ public class SearchTextWithTextPositionFactory { assert sb.length() == stringIdxToPositionIdx.size(); - List positions = sequences.stream()// - .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToUserSpace(textPosition, sequence))).toList(); + List positions = sequences.stream() + .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence))) + .toList(); return SearchTextWithTextPositionModel.builder() .searchText(sb.toString()) @@ -130,7 +132,7 @@ public class SearchTextWithTextPositionFactory { } - private static Rectangle2D mapRedTextPositionToUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) { + private static Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) { float textHeight = sequence.getTextHeight() + HEIGHT_PADDING; Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(), diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextBlockFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextBlockFactory.java index 979e5c1f..0d626763 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextBlockFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextBlockFactory.java @@ -16,7 +16,9 @@ public class TextBlockFactory { AtomicInteger stringOffset; AtomicLong textBlockIdx; + public TextBlockFactory() { + stringOffset = new AtomicInteger(); textBlockIdx = new AtomicLong(); @@ -30,7 +32,11 @@ public class TextBlockFactory { } - public AtomicTextBlock buildAtomicTextBlock(List sequences, SemanticNode parent, DocumentGraphFactory.Context context, Integer numberOnPage, PageNode page) { + public AtomicTextBlock buildAtomicTextBlock(List sequences, + SemanticNode parent, + DocumentGraphFactory.Context context, + Integer numberOnPage, + PageNode page) { SearchTextWithTextPositionModel searchTextWithTextPositionModel = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences); int offset = stringOffset.getAndAdd(searchTextWithTextPositionModel.getSearchText().length()); @@ -51,14 +57,14 @@ public class TextBlockFactory { public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) { - return emptyTextBlock(parent, context, context.pages().get(page).getAndIncrement(), page); + return emptyTextBlock(parent, context.pages().get(page).getAndIncrement(), page); } - public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, Integer numberOnPage, PageNode page) { + public AtomicTextBlock emptyTextBlock(SemanticNode parent, Integer numberOnPage, PageNode page) { return AtomicTextBlock.builder() - .id(textBlockIdx.getAndIncrement()) + .id(textBlockIdx.get()) .boundary(new Boundary(stringOffset.get(), stringOffset.get())) .searchText("") .lineBreaks(Collections.emptyList()) @@ -70,6 +76,4 @@ public class TextBlockFactory { .build(); } - - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java index e1ddcb2c..e638ef4b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java @@ -4,7 +4,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; @@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class FooterNode extends PageElement implements SemanticNode { +public class FooterNode implements SemanticNode { List tocId; TextBlock terminalTextBlock; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java index e257e8b9..1be7cbba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java @@ -4,7 +4,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; @@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class HeaderNode extends PageElement implements SemanticNode { +public class HeaderNode implements SemanticNode { List tocId; TextBlock terminalTextBlock; @@ -44,6 +43,7 @@ public class HeaderNode extends PageElement implements SemanticNode { return terminalTextBlock; } + @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java index 8bf787fd..5da9bb05 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java @@ -4,7 +4,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; @@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class HeadlineNode extends PageElement implements SemanticNode { +public class HeadlineNode implements SemanticNode { List tocId; TextBlock terminalTextBlock; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java index 01d86ec6..b33144ed 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java @@ -4,7 +4,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; @@ -20,7 +19,7 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class ParagraphNode extends PageElement implements SemanticNode { +public class ParagraphNode implements SemanticNode { List tocId; TextBlock terminalTextBlock; @@ -42,9 +41,11 @@ public class ParagraphNode extends PageElement implements SemanticNode { return terminalTextBlock; } + @Override public String toString() { return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.toString(); } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java index 4f67dabc..e3769011 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java @@ -4,7 +4,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; @@ -24,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class SectionNode extends PageElement implements SemanticNode { +public class SectionNode implements SemanticNode { List tocId; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java index cdb82874..3f15a0c5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java @@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; import static java.util.stream.Collectors.groupingBy; import java.awt.geom.Rectangle2D; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -22,6 +21,7 @@ public interface SemanticNode { /** * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity. * So, for a Section all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity + * If the Node is Terminal, the TerminalTextBlock will be returned instead. * * @return TextBlock containing all AtomicTextBlocks that are located under this Node. */ @@ -37,29 +37,42 @@ public interface SemanticNode { Set getEntities(); + /** + * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock + * + * @return Set of PageNodes this node appears on. + */ default Set getPages() { return buildTextBlock().getPages(); } - default PageNode getFirstPage() { - - return getPages().stream().min(Comparator.comparingInt(PageNode::getNumber)).orElseThrow(); - } - - + /** + * @return the TableOfContents of the Document this node belongs to + */ TableOfContents getTableOfContents(); + /** + * The id is a List of Integers uniquely identifying this node in the TableOfContents + * + * @return the TableOfContents ID + */ List getTocId(); + /** + * This should only be used during graph construction + * + * @param tocId List of Integers + */ void setTocId(List tocId); /** - * Traverses the Tree up, until it hits a HeadlineNode or hits a SectionNode which will then return its HeadlineNode. + * Traverses the Tree up, until it hits a HeadlineNode or hits a SectionNode which will then return the first HeadlineNode from its children. + * Throws NotFoundException if no Headline is found this way * * @return First HeadlineNode found */ @@ -69,12 +82,19 @@ public interface SemanticNode { } + /** + * @return boolean indicating wether this Node has a Parent in the TableOfContents + */ default boolean hasParent() { return getTableOfContents().hasParentById(getTocId()); } + /** + * @return The SemanticNode representing the Parent in the TableOfContents + * throws NotFoundException, when no parent is present + */ default SemanticNode getParent() { return getTableOfContents().getParentEntryById(getTocId()).node(); @@ -83,7 +103,8 @@ public interface SemanticNode { /** * Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden. - * Currently only Sections and Tables are not terminal. + * Currently only Sections, Images, and Tables are not terminal. + * A TableCell might be Terminal depending on its area compared to the page. * * @return boolean, indicating if a Node has direct access to a TextBlock */ @@ -122,12 +143,31 @@ public interface SemanticNode { } + /** + * @return true, if this node's TextBlock is not empty + */ + default boolean hasText() { + + return buildTextBlock().length() > 0; + } + + + /** + * @param string A String + * @return true, if this node's TextBlock contains the string + */ default boolean containsString(String string) { return buildTextBlock().getSearchText().contains(string); } + /** + * THis function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity. + * It sets the fields accordingly and recursively calls this function on all its children. + * + * @param entity EntityNode, which is being inserted into the graph + */ default void addThisToEntityIfIntersects(EntityNode entity) { TextBlock textBlock = buildTextBlock(); @@ -143,18 +183,42 @@ public interface SemanticNode { } + /** + * Streams all children located directly underneath this node in the TableOfContents + * + * @return Stream of all children + */ default Stream streamChildren() { return getTableOfContents().streamChildren(getTocId()); } + /** + * recursively streams all SemanticNodes located underneath this node in the TableOfContents in order. + * + * @return Stream of all SubNodes + */ default Stream streamAllSubNodes() { return getTableOfContents().streamSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::node); } + /** + * @return Boundary of this Node's TextBlock + */ + default Boundary getBoundary() { + + return buildTextBlock().getBoundary(); + } + + + /** + * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calcuate the Union of the BoundingBoxes of all Children + * + * @return Rectangle2D fully encapsulating this Node for each page. + */ default Map getBBox() { Map bBoxPerPage = new HashMap<>(); @@ -166,12 +230,10 @@ public interface SemanticNode { } - default Boundary getBoundary() { - - return buildTextBlock().getBoundary(); - } - - + /** + * @param bBoxPerPage initial empty BoundingBox + * @return The union of the BoundingBoxes of all children + */ private Map getBBoxFromChildren(Map bBoxPerPage) { return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> { @@ -181,6 +243,10 @@ public interface SemanticNode { } + /** + * @param bBoxPerPage initial empty BoundingBox + * @return The union of all BoundingBoxes of the TextBlock of this node + */ private Map getBBoxFromTerminalTextBlock(Map bBoxPerPage) { Map> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java index 9e9bdc2e..80de49cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java @@ -8,7 +8,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; @@ -25,7 +24,7 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class TableCellNode extends PageElement implements SemanticNode { +public class TableCellNode implements SemanticNode { List tocId; int row; @@ -60,6 +59,10 @@ public class TableCellNode extends PageElement implements SemanticNode { @Override public TextBlock buildTextBlock() { + if (terminal) { + return terminalTextBlock; + } + if (textBlock == null) { textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector()); } @@ -76,7 +79,7 @@ public class TableCellNode extends PageElement implements SemanticNode { public boolean hasHeader(String headerString) { - return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().contains(headerString)); + return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().strip().equals(headerString)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java index 2c5a2ed6..099eb386 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Stream; -import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; @@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults; @Builder @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class TableNode extends PageElement implements SemanticNode { +public class TableNode implements SemanticNode { List tocId; TableOfContents tableOfContents; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java index 6e4fa403..3023680c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java @@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock; import static java.lang.String.format; import java.awt.geom.Rectangle2D; -import java.util.LinkedList; import java.util.List; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; @@ -121,11 +120,7 @@ public class AtomicTextBlock implements TextBlock { .map(RectangleTransformations::rectangleUnion) .toList(); - List entityPositions = new LinkedList<>(); - for (Rectangle2D position : positionsPerLine) { - entityPositions.add(EntityPosition.builder().position(position).pageNode(page).build()); - } - return entityPositions; + return List.of(EntityPosition.builder().rectanglePerLine(positionsPerLine).pageNode(page).build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java index 0360b62d..8745f271 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java @@ -1,14 +1,18 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock; import static java.lang.String.format; +import static java.util.stream.Collectors.flatMapping; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toList; import java.awt.geom.Rectangle2D; import java.util.LinkedList; import java.util.List; -import java.util.function.Supplier; +import java.util.Map; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import lombok.AccessLevel; import lombok.Data; @@ -16,7 +20,7 @@ import lombok.experimental.FieldDefaults; @Data @FieldDefaults(level = AccessLevel.PRIVATE) -public class ConcatenatedTextBlock implements TextBlock, Supplier { +public class ConcatenatedTextBlock implements TextBlock { List atomicTextBlocks; String searchText; @@ -68,7 +72,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier sb.append(atb.getSearchText())); searchText = sb.toString(); } @@ -96,6 +100,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier getLineBreaks() { @@ -152,14 +157,20 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier mergeEntityPositionsWithSamePageNode(List positions) { + + Map> entityPositionsPerPage = positions.stream().collect(// + groupingBy(EntityPosition::getPageNode, // + flatMapping(entityPosition -> entityPosition.getRectanglePerLine().stream(), toList()))); + + return entityPositionsPerPage.entrySet().stream()// + .map(entry -> EntityPosition.builder().pageNode(entry.getKey()).rectanglePerLine(entry.getValue()).build())// + .toList(); - return this; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java index c9229bd8..c5707f68 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java @@ -99,7 +99,7 @@ public interface TextBlock extends CharSequence { @Override default int length() { - return getSearchText().length(); + return getBoundary().length(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java index 48c69084..18ffc581 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java @@ -13,11 +13,10 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class TextBlockCollector implements Collector { - @Override public Supplier supplier() { - return new ConcatenatedTextBlock(Collections.emptyList()); + return () -> new ConcatenatedTextBlock(Collections.emptyList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java index fea63003..76ca8656 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java @@ -1,7 +1,8 @@ package com.iqser.red.service.redaction.v1.server.document.services; -import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -9,7 +10,6 @@ import java.util.stream.Collectors; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; @@ -30,64 +30,79 @@ public class EntityCreationService { private final EntityTextEnrichmentService entityEnrichmentService; - public Set createEntitiesByLineAfterString(String string, SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) { + public Set createEntitiesByBetweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { + + return Collections.emptySet(); + } + + + public Set createEntitiesBySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) { + + return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary()) + .stream() + .filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary)) + .map(bounds -> createEntityByBoundary(bounds, type, entityType, node)) + .collect(Collectors.toUnmodifiableSet()); + } + + + public Set createEntitiesByLineAfterString(String string, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.buildTextBlock(); SearchImplementation searchImplementation = new SearchImplementation(List.of(string), true); List boundaries = searchImplementation.getBoundaries(textBlock, node.getBoundary()); return boundaries.stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) - .map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph)) + .map(boundary -> createEntityByBoundary(boundary, type, entityType, node)) .collect(Collectors.toUnmodifiableSet()); } - public Set createEntitiesByRegex(String regexPattern, String type, EntityType entityType, DocumentGraph documentGraph) { + public Set createEntitiesByRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) { - List boundaries = RegexMatcher.findBoundaries(regexPattern, documentGraph.buildTextBlock()); - return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph)).collect(Collectors.toUnmodifiableSet()); + List boundaries = RegexMatcher.findBoundaries(regexPattern, node.buildTextBlock()); + return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet()); } - public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, DocumentGraph documentGraph) { + public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType) { + + Boundary boundary = node.buildTextBlock().getBoundary(); + Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1); + return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node); + } + + + public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType); - addEntityToGraph(entity, documentGraph); + addEntityToGraph(entity, node.getTableOfContents()); return entity; } - public EntityNode createEntityByEntity(EntityNode entity, String type, EntityType entityType, DocumentGraph documentGraph) { + public EntityNode createMergedEntity(List entitiesToMerge, String type, EntityType entityType, SemanticNode node) { - return createEntityByBoundary(entity.getBoundary(), type, entityType, documentGraph); - } - - - public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) { - - Boundary boundary = node.buildTextBlock().getBoundary(); - Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1); - return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, documentGraph); - } - - - public Set createEntitiesByString(String entityName, int startOffset, String type, EntityType entityType, DocumentGraph documentGraph) { - - int start = documentGraph.buildTextBlock().indexOf(entityName, startOffset); - if (start == -1) { - throw new NotFoundException(format("Entity %s could no be found!", entityName)); + if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) { + throw new IllegalArgumentException("Provided entities can not be merged!"); } - EntityNode entity = EntityNode.initialEntityNode(new Boundary(start, start + entityName.length()), type, entityType); - addEntityToGraph(entity, documentGraph); - return Set.of(entity); + EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType); + mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction)); + mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet())); + if (mergedEntity.isRedaction()) { + mergedEntity.setRedactionReason(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getRedactionReason).findFirst().orElse("")); + mergedEntity.setMatchedRule(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getMatchedRule).findFirst().orElse(-1)); + mergedEntity.setLegalBasis(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getLegalBasis).findFirst().orElse("")); + } + addEntityToGraph(mergedEntity, node.getTableOfContents()); + return mergedEntity; } - public void addEntityToGraph(EntityNode entity, DocumentGraph documentGraph) { + public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) { try { - SemanticNode containingNode = documentGraph.getTableOfContents() - .getEntries() + SemanticNode containingNode = tableOfContents.getEntries() .stream() .map(TableOfContents.Entry::node) .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary())) @@ -103,7 +118,7 @@ public class EntityCreationService { addToNodeEntitySets(entity); } catch (NotFoundException e) { - entityEnrichmentService.enrichEntity(entity, documentGraph.getTextBlock()); + entityEnrichmentService.enrichEntity(entity, tableOfContents.buildTextBlock()); log.warn("Entity \"{}\" with {} is in between two main sections and will be removed!", entity.getValue(), entity.getBoundary()); entity.removeFromGraph(); } @@ -123,6 +138,27 @@ public class EntityCreationService { entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity)); } + + private static boolean allEntitiesIntersectAndHaveSameTypes(List entitiesToMerge) { + + if (entitiesToMerge.isEmpty()) { + return true; + } + EntityNode previousEntity = entitiesToMerge.get(0); + for (EntityNode entityNode : entitiesToMerge.subList(1, entitiesToMerge.size())) { + boolean typeMatches = entityNode.getType().equals(previousEntity.getType()); + boolean entityTypeMatches = entityNode.getEntityType().equals(previousEntity.getEntityType()); + boolean intersects = entityNode.intersects(previousEntity); + if (typeMatches && entityTypeMatches && intersects) { + previousEntity = entityNode; + continue; + } + return false; + } + return true; + } + + private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) { return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java new file mode 100644 index 00000000..40555733 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java @@ -0,0 +1,32 @@ +package com.iqser.red.service.redaction.v1.server.document.services; + +import java.util.Collection; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; + +public class EntityFieldSetter { + + public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) { + + if (entityNode.getMatchedRule() == -1 && matchedRule != -1) { + entityNode.setMatchedRule(matchedRule); + } + if (entityNode.getRedactionReason().equals("") && redactionReason != null) { + entityNode.setRedactionReason(redactionReason); + } + if (entityNode.getLegalBasis().equals("") && legalBasis != null) { + entityNode.setLegalBasis(legalBasis); + } + if (engine != null) { + entityNode.addEngine(engine); + } + } + + + public static void setFields(Collection entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) { + + entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine)); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java index 173338e8..8b20d1fd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java @@ -1,19 +1,29 @@ package com.iqser.red.service.redaction.v1.server.document.services; +import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators; + import java.util.LinkedList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; public class RegexMatcher { - public static boolean anyMatch(CharSequence searchText, String regexPattern) { + public static boolean anyMatch(CharSequence charSequence, String regexPattern) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); - return pattern.matcher(searchText).find(); + return pattern.matcher(charSequence).find(); + } + + + public static boolean anyMatch(TextBlock textBlock, String regexPattern) { + + Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); + return pattern.matcher(textBlock).region(textBlock.getBoundary().start(), textBlock.getBoundary().end()).find(); } @@ -24,15 +34,16 @@ public class RegexMatcher { return new Boundary(matcher.start(), matcher.end()); } - public static List findBoundaries(String regexPattern, CharSequence searchText) { + + public static List findBoundaries(String regexPattern, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); - Matcher matcher = pattern.matcher(searchText); + Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary())); List boundaries = new LinkedList<>(); while (matcher.find()) { - boundaries.add(new Boundary(matcher.start(), matcher.end())); + boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start())); } - return boundaries; + return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java index 8b13129f..574c5041 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java @@ -1,11 +1,16 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import static java.lang.String.format; + import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.StringUtils; + import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; + import lombok.Data; import lombok.Getter; @@ -50,8 +55,11 @@ public class Dictionary { public DictionaryModel getType(String type) { + DictionaryModel model = localAccessMap.get(type); - if (model == null) throw new NotFoundException("Type: " + type + " is not found"); + if (model == null) { + throw new NotFoundException("Type: " + type + " is not found"); + } return model; } @@ -75,4 +83,23 @@ public class Dictionary { return false; } + + public void addLocalDictionaryEntry(String type, String value, boolean alsoAddLastname) { + + if (localAccessMap.get(type) == null) { + throw new IllegalArgumentException(format("DictionaryModel of type %s does not exist", type)); + } + if (localAccessMap.get(type).getLocalEntries() == null) { + throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type)); + } + if (StringUtils.isEmpty(value) && value.length() < 3) { + throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value)); + } + localAccessMap.get(type).getLocalEntries().add(value); + if (alsoAddLastname) { + String lastname = value.split(" ")[0]; + localAccessMap.get(type).getLocalEntries().add(lastname); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index e5456ac4..f388c625 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -104,10 +104,12 @@ public class DroolsExecutionService { } + @Timed("redactmanager_executeRules") public Section executeRules(KieContainer kieContainer, Section section) { KieSession kieSession = kieContainer.newKieSession(); + kieSession.setGlobal("section", section); kieSession.insert(section); kieSession.fireAllRules(); kieSession.dispose(); @@ -116,7 +118,9 @@ public class DroolsExecutionService { } + public List getEntities(KieSession ks) { + List entities = new LinkedList<>(); QueryResults entitiesResult = ks.getQueryResults("getEntities"); for (QueryResultsRow resultsRow : entitiesResult) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 747d0baf..6f715bc6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -1,7 +1,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle; + import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -14,7 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; @@ -35,19 +36,73 @@ public class RedactionLogCreatorService { private final DictionaryService dictionaryService; - public List createRedactionLog(Map> entityNodesPerPageNode, int numberOfPages, String dossierTemplateId) { + public List createRedactionLog(Set entityNodes, String dossierTemplateId) { List entries = new ArrayList<>(); - entityNodesPerPageNode.forEach(((pageNode, entityNodes) -> entries.addAll(toRedactionLogEntries(pageNode, entityNodes, dossierTemplateId)))); - + Set processedIds = new HashSet<>(); + entityNodes.forEach((entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId)))); return entries; } - private Set toRedactionLogEntries(PageNode pageNode, Set entityNodes, String dossierTemplateId) { + private List toRedactionLogEntries(EntityNode entityNode, Set processedIds, String dossierTemplateId) { - return Collections.emptySet(); + List redactionLogEntities = new ArrayList<>(); + + // Duplicates can exist due table extraction columns over multiple rows. + + for (EntityPosition entityPosition : entityNode.getEntityPositions()) { + + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entityNode, dossierTemplateId); + + if (processedIds.contains(entityPosition.getId())) { + continue; + } + + processedIds.add(entityPosition.getId()); + + redactionLogEntry.setId(entityPosition.getId()); + + List rectanglesPerLine = entityPosition.getRectanglePerLine() + .stream() + .map(rectangle2D -> toRectangle(rectangle2D, entityPosition.getPageNode().getNumber())) + .toList(); + + redactionLogEntry.setPositions(rectanglesPerLine); + } + + return redactionLogEntities; + } + + + private RedactionLogEntry createRedactionLogEntry(EntityNode entity, String dossierTemplateId) { + + Set referenceIds = new HashSet<>(); + entity.getReferences().forEach(ref -> ref.getEntityPositions().forEach(pos -> referenceIds.add(pos.getId()))); + + return RedactionLogEntry.builder() + .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction())) + .reason(entity.getRedactionReason()) + .legalBasis(entity.getLegalBasis()) + .value(entity.getValue()) + .type(entity.getType()) + .redacted(entity.isRedaction()) + .isHint(isHint(entity.getType(), dossierTemplateId)) + .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION)) + .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) + .section(entity.getDeepestFullyContainingNode().getTocId().toString()) + .sectionNumber(entity.getDeepestFullyContainingNode().getTocId().get(0)) + .matchedRule(entity.getMatchedRule()) + .isDictionaryEntry(entity.isDictionaryEntry()) + .textAfter(entity.getTextAfter().toString()) + .textBefore(entity.getTextBefore().toString()) + .startOffset(entity.getBoundary().start()) + .endOffset(entity.getBoundary().end()) + .isDossierDictionaryEntry(entity.isDossierDictionaryEntry()) + .engines(entity.getEngines()) + .reference(referenceIds) + .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 7d428de6..329aad3e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -124,7 +124,7 @@ public final class EntitySearchUtils { orderedEntities.get(i).setPositionSequences(toAdd); } catch (Exception e) { - log.warn("Mismatch between EntityPositionSequence and found Entity!"); + //log.warn("Mismatch between EntityPositionSequence and found Entity!"); } } } @@ -209,9 +209,7 @@ public final class EntitySearchUtils { .length() && inner.getStart() >= outer.getStart() && inner.getEnd() <= outer.getEnd() && outer != inner && outer.getSectionNumber() == inner.getSectionNumber()) { if (outer.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) { wordsToRemove.add(outer); - } else if (!( - inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && outer.getEntityType() == EntityType.ENTITY || - inner.getEntityType() == EntityType.ENTITY && outer.getEntityType() == EntityType.FALSE_RECOMMENDATION)) { + } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && outer.getEntityType() == EntityType.ENTITY || inner.getEntityType() == EntityType.ENTITY && outer.getEntityType() == EntityType.FALSE_RECOMMENDATION)) { if (inner.isResized()) { wordsToRemove.add(outer); } else { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java index 6cc53971..e9a28a93 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java @@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; import java.util.Set; import java.util.regex.Pattern; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; + import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -28,4 +30,23 @@ public final class SeparatorUtils { return intValue >= 12288 && intValue <= 12336 || japaneseAltPunctuationMarks.contains(intValue); } + + + public static boolean validateBoundaryIsSurroundedBySeparators(CharSequence charSequence, Boundary boundary) { + + return validateStart(charSequence, boundary) && validateEnd(charSequence, boundary); + } + + + private static boolean validateEnd(CharSequence charSequence, Boundary boundary) { + + return boundary.end() == charSequence.length() || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end())) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end() - 1)); + } + + + private static boolean validateStart(CharSequence charSequence, Boundary boundary) { + + return boundary.start() == 0 || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start())); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java index 015c82f0..876cdf1c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server; +import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators; import static com.iqser.red.service.redaction.v1.server.utils.PdfDraw.drawRectangle2DList; import java.awt.Color; @@ -18,10 +19,9 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.ClassPathResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition; import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; @@ -29,11 +29,11 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.utils.PdfDraw; @@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { @SneakyThrows public void testDroolsOnDocumentGraph() { - String filename = "files/new/crafted document"; + String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; prepareStorage(filename + ".pdf"); ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); @@ -76,31 +76,39 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); + long dictionarySearchStart = System.currentTimeMillis(); List foundEntities = new LinkedList<>(); - for (var model : dictionary.getDictionaryModels()) { + for (DictionaryModel model : dictionary.getDictionaryModels()) { findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType()); findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType()); findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType()); - foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document)); } + System.out.printf("Dictionary search took %d ms and found %d entities\n", System.currentTimeMillis() - dictionarySearchStart, foundEntities.size()); + + long graphInsertionStart = System.currentTimeMillis(); + foundEntities.forEach(entity -> entity.setDossierDictionaryEntry(true)); + foundEntities.forEach(entity -> entity.setDictionaryEntry(true)); + foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document.getTableOfContents())); + System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart); KieSession kieSession = kieContainer.newKieSession(); - entityCreationService.createEntitiesByString("Michael N.", 0, "CBI_author", EntityType.FALSE_POSITIVE, document)// - .forEach(kieSession::insert); kieSession.setGlobal("document", document); kieSession.setGlobal("entityCreationService", entityCreationService); + kieSession.setGlobal("dictionary", dictionary); + long serializationStart = System.currentTimeMillis(); document.getEntities().forEach(kieSession::insert); document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node())); document.getPages().forEach(kieSession::insert); + System.out.printf("Object serialization and kieSession insertion took %d ms\n", System.currentTimeMillis() - serializationStart); - long start = System.currentTimeMillis(); + long dictionaryAddsStart = System.currentTimeMillis(); kieSession.insert(FileAttribute.builder().label("Vertebrate Study").value("Yes").build()); - + kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); kieSession.fireAllRules(); - System.out.printf("Firing all rules took %d ms\n", System.currentTimeMillis() - start); - - //List redactionLogEntries = redactionLogCreatorService.createRedactionLog(); + System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - dictionaryAddsStart); + System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart); + List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document.getEntities(), TEST_DOSSIER_TEMPLATE_ID); drawAllEntities(filename, fileResource, document); } @@ -181,7 +189,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { var insertStart = System.currentTimeMillis(); DocumentGraph finalDocument = document; - foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument)); + foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument.getTableOfContents())); var insertTime = System.currentTimeMillis() - insertStart; totalInsertTime += insertTime; @@ -210,7 +218,8 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { .filter(entityNode -> !entityNode.isRemoved()) .filter(EntityNode::isRedaction) .flatMap(entityNode -> entityNode.getEntityPositions().stream()) - .map(EntityPosition::getPosition) + .filter(entityPosition -> entityPosition.getPageNode().equals(page)) + .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream()) .toList(); PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLACK).stroke(true).build(); @@ -221,9 +230,10 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { List entityPositionsOnPage = page.getEntities() .stream() .filter(entityNode -> !entityNode.isRemoved()) - .filter(e -> !e.isRedaction()) + .filter(entityNode -> !entityNode.isRedaction()) .flatMap(entityNode -> entityNode.getEntityPositions().stream()) - .map(EntityPosition::getPosition) + .filter(entityPosition -> entityPosition.getPageNode().equals(page)) + .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream()) .toList(); PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLUE).stroke(true).build(); @@ -245,16 +255,9 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { TextBlock textBlock = documentGraph.getTextBlock(); searchImplementation.getBoundaries(textBlock, textBlock.getBoundary()) .stream() - .filter(boundary -> validateSeparators(textBlock, boundary)) + .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)) .map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType)) .forEach(foundEntities::add); } - - private static boolean validateSeparators(TextBlock textBlock, Boundary boundary) { - - return (boundary.start() == 0 || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start()))) && (boundary.end() == textBlock.length() || SeparatorUtils.isSeparator( - textBlock.charAt(boundary.end())) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1))); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java index bb915854..474ac517 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java @@ -7,8 +7,8 @@ import org.springframework.core.io.ClassPathResource; import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentDataMapper; -import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; @@ -46,11 +46,6 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { assert document.toString().equals(newDocumentGraph.toString()); assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString()); - for (int pageIndex = 1; pageIndex < document.getNumberOfPages(); pageIndex++) { - var page = document.getPages().get(pageIndex); - var newPage = document.getPages().get(pageIndex); - assert page.toString().equals(newPage.toString()); - } } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java index 1514b524..4186b6d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java @@ -14,16 +14,16 @@ import org.springframework.core.io.ClassPathResource; import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries; import com.iqser.red.service.redaction.v1.server.classification.model.Document; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; @@ -56,7 +56,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("Expand to Hint ", entityNode.getTextBefore()); assertEquals("’s Donut ←", entityNode.getTextAfter()); @@ -80,7 +80,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("", entityNode.getTextBefore()); assertEquals(" Purity Hint", entityNode.getTextAfter()); @@ -103,7 +103,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("", entityNode.getTextBefore()); assertEquals("", entityNode.getTextAfter()); @@ -189,7 +189,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore()); assertEquals(", Group 9;", entityNode.getTextAfter()); @@ -238,7 +238,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore()); assertEquals(" of metabolite of", entityNode.getTextAfter()); @@ -293,8 +293,8 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); - PageNode pageNode = documentGraph.getPages().get(pageNumber - 1); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow(); assertEquals(entityNode.getValue(), searchTerm); assertTrue(pageNode.getEntities().contains(entityNode)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl index 2313b821..879cebbb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl @@ -2,6 +2,7 @@ package drools import static java.lang.String.format; import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch; +import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields; import java.util.List; @@ -17,9 +18,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu import java.util.Set import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; global DocumentGraph document global EntityCreationService entityCreationService +global Dictionary dictionary + +// --------------------------------------- CBI rules ------------------------------------------------------------------- rule "1: Redact CBI Authors (Non vertebrate study)" no-loop true @@ -29,14 +34,7 @@ rule "1: Redact CBI Authors (Non vertebrate study)" $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then $entity.setRedaction(true); - if ($entity.getMatchedRule() == -1) { - $entity.setMatchedRule(1); - } - if ($entity.getRedactionReason().equals("")) { - $entity.setRedactionReason("Author found"); - } - $entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); - $entity.addEngine(Engine.RULE); + setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); update($entity) end @@ -48,14 +46,7 @@ rule "2: Redact CBI Authors (Vertebrate study)" $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then $entity.setRedaction(true); - if ($entity.getMatchedRule() == -1) { - $entity.setMatchedRule(2); - } - if ($entity.getRedactionReason().equals("")) { - $entity.setRedactionReason("Author found"); - } - $entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); - $entity.addEngine(Engine.RULE); + setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); update($entity) end @@ -65,15 +56,9 @@ rule "3: Don't redact CBI Address (Non vertebrate study)" when FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) - $recommendationEntity: EntityNode(type == "CBI_address", entityType == EntityType.RECOMMENDATION) then - $entity.setRedaction(false); - $entity.setMatchedRule(3); - $entity.setRedactionReason("Address found for non vertebrate study"); - $entity.addEngine(Engine.RULE); + setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); update($entity) - $recommendationEntity.removeFromGraph(); - delete($recommendationEntity); end rule "4: Redact CBI Address (Vertebrate study)" @@ -84,43 +69,43 @@ rule "4: Redact CBI Address (Vertebrate study)" $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) then $entity.setRedaction(true); - $entity.setMatchedRule(4); - $entity.setRedactionReason("Address found"); - $entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); - $entity.addEngine(Engine.RULE); + setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); update($entity) end rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" when - $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true) + $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction) then - EntityNode entity = entityCreationService.createEntityByEntity($entity, "CBI_author", EntityType.FALSE_POSITIVE, document); - entity.setMatchedRule(5); + EntityNode entity = entityCreationService.createEntityByBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); + setFields($entity, 5, "Genitive Author", null, Engine.RULE); insert(entity); end -rule "6: Create Entity from Author(s) cells in Tables with Author(s) header" +rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation" + agenda-group "LOCAL_DICTIONARY_ADDS" when - authorCell: TableCellNode(header == false, (hasHeader("Author(s)") || hasHeader("Author"))) + authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText()) then - EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY, document); - entity.setMatchedRule(6); - entity.setRedactionReason("Header \"Author(s)\" found"); + EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY); + setFields(entity, 6, "Header Author(s) found", null, Engine.RULE); insert(entity); + dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true); end rule "7: Add CBI_author with \"et al.\" Regex" + agenda-group "LOCAL_DICTIONARY_ADDS" when + $section: SectionNode(containsString("et al.")) then - Set entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, document); - entities.forEach(entity -> entity.setMatchedRule(7)); - entities.forEach(entity -> entity.setRedactionReason("Found by \"et al.\" regex")); + Set entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section); + setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE); entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false)); end rule "8: Add recommendation for Addresses in Test Organism sections" @@ -129,9 +114,8 @@ rule "8: Add recommendation for Addresses in Test Organism sections" FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $section: SectionNode(containsString("Species") && containsString("Source")) then - Set entities = entityCreationService.createEntitiesByLineAfterString("Source", $section, "CBI_address", EntityType.RECOMMENDATION, document); - entities.forEach(entity -> entity.setRedactionReason("Line after \"Source\" in Test Organism Section")); - entities.forEach(entity -> entity.setMatchedRule(8)); + Set entities = entityCreationService.createEntitiesByLineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE); entities.forEach(entity -> insert(entity)); end @@ -142,9 +126,46 @@ rule "9: Add recommendation for Addresses in Test Animals sections" FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $section: SectionNode(containsString("Species:") && containsString("Source:")) then - Set entities = entityCreationService.createEntitiesByLineAfterString("Source:", $section, "CBI_address", EntityType.RECOMMENDATION, document); - entities.forEach(entity -> entity.setRedactionReason("Line after \"Source:\" in Test Organism Section")); - entities.forEach(entity -> entity.setMatchedRule(9)); + Set entities = entityCreationService.createEntitiesByLineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE); entities.forEach(entity -> insert(entity)); end +// --------------------------------------- PII rules ------------------------------------------------------------------- + + +rule "10: Redacted PII Personal Identification Information (Non vertebrate study)" + no-loop true + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY) + then + $entity.setRedaction(true); + setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) + end + + + +rule "11: Redacted PII Personal Identification Information (Vertebrate study)" + no-loop true + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY) + then + $entity.setRedaction(true); + setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) + end + +rule "12: Redact Emails by RegEx (Non vertebrate study)" + + when + $section: SectionNode(containsString("@")) + then + Set entities = entityCreationService.createEntitiesByRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 12, "Found by email regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl index 325dd392..f28c0fe1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl @@ -8,75 +8,88 @@ import java.util.List; import java.util.LinkedList; import java.util.HashSet; +import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.document.graph.* import com.iqser.red.service.redaction.v1.server.document.graph.nodes.* import com.iqser.red.service.redaction.v1.server.document.graph.entity.* import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import java.util.Set; +import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; global DocumentGraph document +global EntityCreationService entityCreationService +global Dictionary dictionary -rule "merge contained Entities of same type" +rule "merge intersecting Entities of same type" salience 100 + when - outer: EntityNode($type: type, $entityType: entityType) - inner: EntityNode(this != outer, containedBy(outer), type == $type, entityType == $entityType) + $first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) + $second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) then - System.out.printf("removed entity %s contained by %s\n", inner, outer); - outer.addEngines(inner.getEngines()); - inner.removeFromGraph(); - delete(inner); + $first.removeFromGraph(); + $second.removeFromGraph(); + EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document); + insert(mergedEntity); + delete($first); + delete($second); end rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 100 + when $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE) - entity: EntityNode(this != $falsePositive, containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY) + entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then - System.out.printf("removed entity %s marked false positive by %s\n", entity, $falsePositive); entity.removeFromGraph(); delete(entity); end -/* -rule "If same type, remove inner" +rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 100 + when - $containedEntity: ContainedEntity($toRemove: inner, outer.getType() == inner.getType()) + $falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION) + $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $toRemove.removeFromGraph(); - delete($toRemove); - delete($containedEntity); + $recommendation.removeFromGraph(); + delete($recommendation); end -rule "If outer Recommended and inner Entity, remove outer" - salience 99 +rule "remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 100 + when - $containedEntity: ContainedEntity($toRemove: outer, outer.getType() == EntityType.RECOMMENDATION, inner.getType() == EntityType.ENTITY) + $entity: EntityNode($type: type, entityType == EntityType.ENTITY) + $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - $toRemove.removeFromGraph(); - delete($toRemove); - delete($containedEntity); + $recommendation.removeFromGraph(); + delete($recommendation); end -rule "Inner not False Recommendation and outer not Entity" - salience 99 +rule "remove Entity of lower rank, when equal boundaries and entityType" + salience 100 + when - $containedEntity: ContainedEntity($toRemove: outer, (outer.getType() != EntityType.ENTITY || inner.getType() != EntityType.FALSE_RECOMMENDATION)) + $higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary) + $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type)) then - $toRemove.removeFromGraph(); - delete($toRemove); - delete($containedEntity); - end -rule "Remove leftover ContainedEntities" - salience 0 - when - $toRemove: ContainedEntity() - then - delete($toRemove); + $lowerRank.removeFromGraph(); + delete($lowerRank); end -*/ +rule "run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + + when + DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() + then + Set entityNodes = entityCreationService.createEntitiesBySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document); + System.out.printf("local dictionary search found %d entities\n", entityNodes.size()); + entityNodes.forEach(entityNode -> insert(entityNode)); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index e69de29b..9dd68653 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -0,0 +1,367 @@ +package drools + +import com.iqser.red.service.redaction.v1.server.redaction.model.Section + +global Section section + + +// --------------------------------------- AI rules ------------------------------------------------------------------- + +rule "0: Add CBI_author from ai" + when + Section(aiMatchesType("CBI_author")) + then + section.addAiEntities("CBI_author", "CBI_author"); + end + +rule "0: Combine address parts from ai to CBI_address (org is mandatory)" + when + Section(aiMatchesType("ORG")) + then + section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); + end + +rule "0: Combine address parts from ai to CBI_address (street is mandatory)" + when + Section(aiMatchesType("STREET")) + then + section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); + end + +rule "0: Combine address parts from ai to CBI_address (city is mandatory)" + when + Section(aiMatchesType("CITY")) + then + section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false); + end + + +// --------------------------------------- CBI rules ------------------------------------------------------------------- + +rule "1: Redact CBI Authors (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + then + section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "2: Redact CBI Authors (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + then + section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "3: Redact not CBI Address (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + then + section.redactNot("CBI_address", 3, "Address found for non vertebrate study"); + section.ignoreRecommendations("CBI_address"); + end + +rule "4: Redact CBI Address (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + then + section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "5: Do not redact genitive CBI_author" + when + Section(matchesType("CBI_author")) + then + section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); + end + + +rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) + then + section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) + then + section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) + then + section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "14: Redact and add recommendation for et al. author (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) + then + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "15: Redact and add recommendation for et al. author (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) + then + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "16: Add recommendation for Addresses in Test Organism sections" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:")) + then + section.recommendLineAfter("Source:", "CBI_address"); + end + +rule "17: Add recommendation for Addresses in Test Animals sections" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source")) + then + section.recommendLineAfter("Source", "CBI_address"); + end + + +rule "18: Do not redact Names and Addresses if Published Information found" + when + Section(matchesType("published_information")) + then + section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found"); + section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found"); + end + + +// --------------------------------------- PII rules ------------------------------------------------------------------- + + +rule "19: Redacted PII Personal Identification Information (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) + then + section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "20: Redacted PII Personal Identification Information (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) + then + section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "21: Redact Emails by RegEx (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) + then + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "22: Redact Emails by RegEx (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) + then + section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "23: Redact contact information (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") + || text.contains("Contact:") + || text.contains("Alternative contact:") + || (text.contains("No:") && text.contains("Fax")) + || (text.contains("Contact:") && text.contains("Tel.:")) + || text.contains("European contact:") + )) + then + section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "24: Redact contact information (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") + || text.contains("Contact:") + || text.contains("Alternative contact:") + || (text.contains("No:") && text.contains("Fax")) + || (text.contains("Contact:") && text.contains("Tel.:")) + || text.contains("European contact:") + )) + then + section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( + text.contains("Contact") + || text.contains("Telephone") + || text.contains("Phone") + || text.contains("Fax") + || text.contains("Tel") + || text.contains("Ter") + || text.contains("Mobile") + || text.contains("Fel") + || text.contains("Fer") + )) + then + section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( + text.contains("Contact") + || text.contains("Telephone") + || text.contains("Phone") + || text.contains("Fax") + || text.contains("Tel") + || text.contains("Ter") + || text.contains("Mobile") + || text.contains("Fel") + || text.contains("Fer") + )) + then + section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "27: Redact AUTHOR(S) (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("AUTHOR(S):") + && searchText.contains("COMPLETION DATE:") + && !searchText.contains("STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "28: Redact AUTHOR(S) (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("AUTHOR(S):") + && searchText.contains("COMPLETION DATE:") + && !searchText.contains("STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "29: Redact AUTHOR(S) (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("AUTHOR(S):") + && searchText.contains("STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "30: Redact AUTHOR(S) (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("AUTHOR(S):") + && searchText.contains("STUDY COMPLETION DATE:") + ) + then + section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("PERFORMING LABORATORY:") + ) + then + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); + end + +rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") + && searchText.contains("PERFORMING LABORATORY:")) + then + section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + +rule "33: Redact study director abbreviation" + when + Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML"))) + then + section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + + +// --------------------------------------- other rules ------------------------------------------------------------------- + +rule "34: Purity Hint" + when + Section(searchText.toLowerCase().contains("purity")) + then + section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); + end + + + +rule "35: Redact signatures (Non vertebrate study)" + when + Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) + then + section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "36: Redact signatures (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) + then + section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +rule "43: Redact Logos (Vertebrate study)" + when + Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo")) + then + section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end