RED-6369: Rules Refactor

*added some new Rules and Methods to support them
*integrated local dictionary searches into ruleflow using agenda-groups
*Generalized methods to be able to execute on any SemanticNode
*made Documentgraph implement SemanticNode

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java

@@ -20,7 +20,7 @@ public class DocumentDataMapper {
 
     public DocumentData toDocumentData(DocumentGraph documentGraph) {
 
-        List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTextBlocksInOrder()
+        List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder()
                 .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
                 .distinct()
                 .map(this::toAtomicTextBlockData)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java

@@ -22,12 +22,12 @@ import com.iqser.red.service.redaction.v1.server.document.data.PageData;
 import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
@@ -44,7 +44,11 @@ public class DocumentGraphMapper {
 
         buildNodesFromTableOfContents(Collections.emptyList(), context);
 
-        DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(documentData.getPages().size()).pages(context.pages).tableOfContents(context.tableOfContents).build();
+        DocumentGraph documentGraph = DocumentGraph.builder()
+                .numberOfPages(documentData.getPages().size())
+                .pages(new HashSet<>(context.pages))
+                .tableOfContents(context.tableOfContents)
+                .build();
         documentGraph.setTextBlock(documentGraph.buildTextBlock());
         return documentGraph;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java

@@ -86,7 +86,7 @@ public class Boundary implements Comparable<Boundary> {
 
     public List<Boundary> split(List<Integer> splitIndices) {
 
-        if(splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
+        if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
             throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this));
         }
         List<Boundary> splitBoundaries = new LinkedList<>();
@@ -100,6 +100,14 @@ public class Boundary implements Comparable<Boundary> {
     }
 
 
+    public static Boundary merge(List<Boundary> boundaries) {
+
+        int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
+        int maxEnd = boundaries.stream().mapToInt(Boundary::end).max().orElseThrow(IllegalArgumentException::new);
+        return new Boundary(minStart, maxEnd);
+    }
+
+
     @Override
     public String toString() {
 
@@ -109,6 +117,7 @@ public class Boundary implements Comparable<Boundary> {
 
     @Override
     public int compareTo(Boundary boundary) {
+
         if (end < boundary.end() && start < boundary.start()) {
             return -1;
         }
@@ -119,12 +128,14 @@ public class Boundary implements Comparable<Boundary> {
         return 0;
     }
 
+
     @Override
     public int hashCode() {
 
         return toString().hashCode();
     }
 
+
     @Override
     public boolean equals(Object object) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java

@@ -1,14 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.document.graph;
 
+import java.awt.geom.Rectangle2D;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
@@ -24,9 +28,9 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentGraph {
+public class DocumentGraph implements SemanticNode {
 
-    List<PageNode> pages;
+    Set<PageNode> pages;
     TableOfContents tableOfContents;
     Integer numberOfPages;
     TextBlock textBlock;
@@ -34,7 +38,7 @@ public class DocumentGraph {
 
     public TextBlock buildTextBlock() {
 
-        return streamTextBlocksInOrder().collect(new TextBlockCollector());
+        return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
     }
 
 
@@ -44,7 +48,7 @@ public class DocumentGraph {
     }
 
 
-    public Stream<TextBlock> streamTextBlocksInOrder() {
+    public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
 
         return streamAllNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock);
     }
@@ -56,6 +60,20 @@ public class DocumentGraph {
     }
 
 
+    @Override
+    public List<Integer> getTocId() {
+
+        return Collections.emptyList();
+    }
+
+
+    @Override
+    public void setTocId(List<Integer> tocId) {
+
+        throw new UnsupportedOperationException("DocumentGraph is always the root of the Table of Contents");
+    }
+
+
     private Stream<SemanticNode> streamAllNodes() {
 
         return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node);
@@ -68,4 +86,15 @@ public class DocumentGraph {
         return tableOfContents.toString();
     }
 
+
+    @Override
+    public Map<PageNode, Rectangle2D> getBBox() {
+
+        Map<PageNode, Rectangle2D> bBox = new HashMap<>();
+        for (PageNode page : pages) {
+            bBox.put(page, new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight()));
+        }
+        return bBox;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageElement.java

@@ -1,14 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.graph;
-
-import java.util.Map;
-
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
-
-import lombok.Data;
-
-@Data
-public class PageElement {
-
-    Map<Integer, RedRectangle2D> positionsPerPage;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java

@@ -11,8 +11,11 @@ import java.util.List;
 import java.util.stream.Stream;
 
 import com.google.common.hash.Hashing;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 
 import lombok.Builder;
 import lombok.Data;
@@ -29,6 +32,12 @@ public class TableOfContents {
     }
 
 
+    public TextBlock buildTextBlock() {
+
+        return streamEntriesInOrder().map(Entry::node).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
+    }
+
+
     public List<Integer> createNewEntryAndReturnId(NodeType nodeType, SemanticNode node) {
 
         return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node);
@@ -72,7 +81,7 @@ public class TableOfContents {
 
         List<Integer> parentIds = getParentId(tocId);
         if (parentIds.size() < 1) {
-            throw new UnsupportedOperationException(format("Node with tocId \"%s\" has no parent!", tocId));
+            throw new NotFoundException(format("Node with tocId \"%s\" has no parent!", tocId));
         }
         return getEntryById(parentIds);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java

@@ -32,7 +32,6 @@ public class EntityNode {
                 .entityType(entityType)
                 .boundary(boundary)
                 .redaction(false)
-                .falsePositive(false)
                 .removed(false)
                 .ignored(false)
                 .resized(false)
@@ -55,7 +54,6 @@ public class EntityNode {
 
     // empty defaults
     boolean redaction;
-    boolean falsePositive;
     boolean removed;
     boolean ignored;
     boolean resized;
@@ -117,17 +115,30 @@ public class EntityNode {
     }
 
 
+    public boolean intersects(EntityNode entityNode) {
+
+        return this.boundary.intersects(entityNode.getBoundary());
+    }
+
+
     public void addEngine(Engine engine) {
 
         engines.add(engine);
     }
 
+
     public void addEngines(Set<Engine> engines) {
 
         this.engines.addAll(engines);
     }
 
 
+    public void addReference(EntityNode entityNode) {
+
+        references.add(entityNode);
+    }
+
+
     @Override
     public String toString() {
 
@@ -157,6 +168,7 @@ public class EntityNode {
         return Hashing.murmur3_128().hashString(toString(), StandardCharsets.UTF_8).hashCode();
     }
 
+
     @Override
     public boolean equals(Object o) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph.entity;
 
 import java.awt.geom.Rectangle2D;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 
 import com.google.common.hash.Hashing;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
@@ -17,7 +18,7 @@ import lombok.experimental.FieldDefaults;
 public class EntityPosition {
 
     PageNode pageNode;
-    Rectangle2D position;
+    List<Rectangle2D> rectanglePerLine;
 
 
     public String getId() {
@@ -29,8 +30,10 @@ public class EntityPosition {
     @Override
     public int hashCode() {
 
-        String string = String.valueOf(pageNode.getNumber()) + position.getX() + position.getY() + position.getWidth() + position.getHeight();
+        StringBuilder sb = new StringBuilder();
-        return Hashing.murmur3_128().hashString(string, StandardCharsets.UTF_8).hashCode();
+        sb.append(pageNode.getNumber());
+        rectanglePerLine.forEach(r -> sb.append(r.getX()).append(r.getY()).append(r.getWidth()).append(r.getHeight()));
+        return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java

@@ -8,7 +8,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
@@ -30,7 +29,7 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class ImageNode extends PageElement implements SemanticNode {
+public class ImageNode implements SemanticNode {
 
     List<Integer> tocId;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java

@@ -10,7 +10,6 @@ import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toList;
 
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -28,16 +27,16 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
@@ -71,11 +70,7 @@ public class DocumentGraphFactory {
         addSections(document, context);
         addHeaderAndFooterToEachPage(document, context);
 
-        DocumentGraph documentGraph = DocumentGraph.builder()
+        DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(context.pages.size()).pages(context.pages.keySet()).tableOfContents(context.tableOfContents).build();
-                .numberOfPages(context.pages.size())
-                .pages(context.pages.keySet().stream().sorted(Comparator.comparingInt(PageNode::getNumber)).toList())
-                .tableOfContents(context.tableOfContents)
-                .build();
 
         documentGraph.setTextBlock(documentGraph.buildTextBlock());
         return documentGraph;
@@ -193,28 +188,44 @@ public class DocumentGraphFactory {
         tableCellNode.setTocId(tocId);
 
         if (cell.getTextBlocks().isEmpty()) {
-            tableCellNode.setTerminal(false);
+            tableCellNode.setTerminalTextBlock(context.textBlockFactory.emptyTextBlock(parentNode, context, page));
-            return;
+            tableCellNode.setTerminal(true);
-        }
 
-        if (cell.getTextBlocks().size() == 1) {
+        } else if (cell.getTextBlocks().size() == 1) {
             textBlock = context.textBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCellNode, context, page);
             tableCellNode.setTerminalTextBlock(textBlock);
-        } else if (StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H")) {
+            tableCellNode.setTerminal(true);
-            tableCellNode.setTerminal(false);
+
+        } else if (firstTextBlockIsHeadline(cell)) {
             addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractTextContainer) tb).toList(), Collections.emptyList(), context);
-        } else if (cell.getArea() < TABLE_CELL_MERGE_SIZE_THRESHOLD * page.getHeight() * page.getWidth()) {
+            tableCellNode.setTerminal(false);
+
+        } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
             List<TextPositionSequence> sequences = mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
             textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page);
             tableCellNode.setTerminalTextBlock(textBlock);
+            tableCellNode.setTerminal(true);
+
         } else {
-            tableCellNode.setTerminal(false);
             cell.getTextBlocks().forEach(tb -> addParagraphOrHeadline(tableCellNode, tb, context));
+            tableCellNode.setTerminal(false);
         }
 
     }
 
 
+    private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, PageNode page) {
+
+        return cell.getArea() < TABLE_CELL_MERGE_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
+    }
+
+
+    private static boolean firstTextBlockIsHeadline(Cell cell) {
+
+        return StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H");
+    }
+
+
     private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context) {
 
         addParagraphOrHeadline(parentNode, originalTextBlock, context, Collections.emptyList());
@@ -340,7 +351,7 @@ public class DocumentGraphFactory {
 
         PageNode page = getPage(pageIndex, context);
         HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, context, 0, page);
+        AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
         List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header);
         header.setTocId(tocId);
         header.setTerminalTextBlock(textBlock);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java

@@ -10,6 +10,8 @@ import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collector;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@@ -52,6 +54,12 @@ public class RectangleTransformations {
     }
 
 
+    public static Rectangle toRectangle(Rectangle2D rectangle2D, int pageNumber) {
+
+        return new Rectangle(new Point((float) rectangle2D.getMaxX(), (float) rectangle2D.getMaxY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
+    }
+
+
     private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java

@@ -18,6 +18,7 @@ public class SearchTextWithTextPositionFactory {
 
     public static final int HEIGHT_PADDING = 2;
 
+
     public static SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
 
         if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
@@ -79,8 +80,9 @@ public class SearchTextWithTextPositionFactory {
 
         assert sb.length() == stringIdxToPositionIdx.size();
 
-        List<Rectangle2D> positions = sequences.stream()//
+        List<Rectangle2D> positions = sequences.stream()
-                .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToUserSpace(textPosition, sequence))).toList();
+                .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence)))
+                .toList();
 
         return SearchTextWithTextPositionModel.builder()
                 .searchText(sb.toString())
@@ -130,7 +132,7 @@ public class SearchTextWithTextPositionFactory {
     }
 
 
-    private static Rectangle2D mapRedTextPositionToUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
+    private static Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
 
         float textHeight = sequence.getTextHeight() + HEIGHT_PADDING;
         Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(),

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/TextBlockFactory.java

@@ -16,7 +16,9 @@ public class TextBlockFactory {
     AtomicInteger stringOffset;
     AtomicLong textBlockIdx;
 
+
     public TextBlockFactory() {
+
         stringOffset = new AtomicInteger();
         textBlockIdx = new AtomicLong();
 
@@ -30,7 +32,11 @@ public class TextBlockFactory {
     }
 
 
-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Integer numberOnPage, PageNode page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences,
+                                                SemanticNode parent,
+                                                DocumentGraphFactory.Context context,
+                                                Integer numberOnPage,
+                                                PageNode page) {
 
         SearchTextWithTextPositionModel searchTextWithTextPositionModel = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
         int offset = stringOffset.getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
@@ -51,14 +57,14 @@ public class TextBlockFactory {
 
     public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) {
 
-        return emptyTextBlock(parent, context, context.pages().get(page).getAndIncrement(), page);
+        return emptyTextBlock(parent, context.pages().get(page).getAndIncrement(), page);
     }
 
 
-    public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, Integer numberOnPage, PageNode page) {
+    public AtomicTextBlock emptyTextBlock(SemanticNode parent, Integer numberOnPage, PageNode page) {
 
         return AtomicTextBlock.builder()
-                .id(textBlockIdx.getAndIncrement())
+                .id(textBlockIdx.get())
                 .boundary(new Boundary(stringOffset.get(), stringOffset.get()))
                 .searchText("")
                 .lineBreaks(Collections.emptyList())
@@ -70,6 +76,4 @@ public class TextBlockFactory {
                 .build();
     }
 
-
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java

@@ -4,7 +4,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class FooterNode extends PageElement implements SemanticNode {
+public class FooterNode implements SemanticNode {
 
     List<Integer> tocId;
     TextBlock terminalTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java

@@ -4,7 +4,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class HeaderNode extends PageElement implements SemanticNode {
+public class HeaderNode implements SemanticNode {
 
     List<Integer> tocId;
     TextBlock terminalTextBlock;
@@ -44,6 +43,7 @@ public class HeaderNode extends PageElement implements SemanticNode {
         return terminalTextBlock;
     }
 
+
     @Override
     public String toString() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java

@@ -4,7 +4,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class HeadlineNode extends PageElement implements SemanticNode {
+public class HeadlineNode implements SemanticNode {
 
     List<Integer> tocId;
     TextBlock terminalTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java

@@ -4,7 +4,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@@ -20,7 +19,7 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class ParagraphNode extends PageElement implements SemanticNode {
+public class ParagraphNode implements SemanticNode {
 
     List<Integer> tocId;
     TextBlock terminalTextBlock;
@@ -42,9 +41,11 @@ public class ParagraphNode extends PageElement implements SemanticNode {
         return terminalTextBlock;
     }
 
+
     @Override
     public String toString() {
 
         return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.toString();
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java

@@ -4,7 +4,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@@ -24,7 +23,7 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class SectionNode extends PageElement implements SemanticNode {
+public class SectionNode implements SemanticNode {
 
     List<Integer> tocId;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 import static java.util.stream.Collectors.groupingBy;
 
 import java.awt.geom.Rectangle2D;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -22,6 +21,7 @@ public interface SemanticNode {
     /**
      * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity.
      * So, for a Section all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity
+     * If the Node is Terminal, the TerminalTextBlock will be returned instead.
      *
      * @return TextBlock containing all AtomicTextBlocks that are located under this Node.
      */
@@ -37,29 +37,42 @@ public interface SemanticNode {
     Set<EntityNode> getEntities();
 
 
+    /**
+     * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock
+     *
+     * @return Set of PageNodes this node appears on.
+     */
     default Set<PageNode> getPages() {
 
         return buildTextBlock().getPages();
     }
 
 
-    default PageNode getFirstPage() {
+    /**
-
+     * @return the TableOfContents of the Document this node belongs to
-        return getPages().stream().min(Comparator.comparingInt(PageNode::getNumber)).orElseThrow();
+     */
-    }
-
-
     TableOfContents getTableOfContents();
 
 
+    /**
+     * The id is a List of Integers uniquely identifying this node in the TableOfContents
+     *
+     * @return the TableOfContents ID
+     */
     List<Integer> getTocId();
 
 
+    /**
+     * This should only be used during graph construction
+     *
+     * @param tocId List of Integers
+     */
     void setTocId(List<Integer> tocId);
 
 
     /**
-     * Traverses the Tree up, until it hits a HeadlineNode or hits a SectionNode which will then return its HeadlineNode.
+     * Traverses the Tree up, until it hits a HeadlineNode or hits a SectionNode which will then return the first HeadlineNode from its children.
+     * Throws NotFoundException if no Headline is found this way
      *
      * @return First HeadlineNode found
      */
@@ -69,12 +82,19 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * @return boolean indicating wether this Node has a Parent in the TableOfContents
+     */
     default boolean hasParent() {
 
         return getTableOfContents().hasParentById(getTocId());
     }
 
 
+    /**
+     * @return The SemanticNode representing the Parent in the TableOfContents
+     * throws NotFoundException, when no parent is present
+     */
     default SemanticNode getParent() {
 
         return getTableOfContents().getParentEntryById(getTocId()).node();
@@ -83,7 +103,8 @@ public interface SemanticNode {
 
     /**
      * Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden.
-     * Currently only Sections and Tables are not terminal.
+     * Currently only Sections, Images, and Tables are not terminal.
+     * A TableCell might be Terminal depending on its area compared to the page.
      *
      * @return boolean, indicating if a Node has direct access to a TextBlock
      */
@@ -122,12 +143,31 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * @return true, if this node's TextBlock is not empty
+     */
+    default boolean hasText() {
+
+        return buildTextBlock().length() > 0;
+    }
+
+
+    /**
+     * @param string A String
+     * @return true, if this node's TextBlock contains the string
+     */
     default boolean containsString(String string) {
 
         return buildTextBlock().getSearchText().contains(string);
     }
 
 
+    /**
+     * THis function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity.
+     * It sets the fields accordingly and recursively calls this function on all its children.
+     *
+     * @param entity EntityNode, which is being inserted into the graph
+     */
     default void addThisToEntityIfIntersects(EntityNode entity) {
 
         TextBlock textBlock = buildTextBlock();
@@ -143,18 +183,42 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * Streams all children located directly underneath this node in the TableOfContents
+     *
+     * @return Stream of all children
+     */
     default Stream<SemanticNode> streamChildren() {
 
         return getTableOfContents().streamChildren(getTocId());
     }
 
 
+    /**
+     * recursively streams all SemanticNodes located underneath this node in the TableOfContents in order.
+     *
+     * @return Stream of all SubNodes
+     */
     default Stream<SemanticNode> streamAllSubNodes() {
 
         return getTableOfContents().streamSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::node);
     }
 
 
+    /**
+     * @return Boundary of this Node's TextBlock
+     */
+    default Boundary getBoundary() {
+
+        return buildTextBlock().getBoundary();
+    }
+
+
+    /**
+     * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calcuate the Union of the BoundingBoxes of all Children
+     *
+     * @return Rectangle2D fully encapsulating this Node for each page.
+     */
     default Map<PageNode, Rectangle2D> getBBox() {
 
         Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
@@ -166,12 +230,10 @@ public interface SemanticNode {
     }
 
 
-    default Boundary getBoundary() {
+    /**
-
+     * @param bBoxPerPage initial empty BoundingBox
-        return buildTextBlock().getBoundary();
+     * @return The union of the BoundingBoxes of all children
-    }
+     */
-
-
     private Map<PageNode, Rectangle2D> getBBoxFromChildren(Map<PageNode, Rectangle2D> bBoxPerPage) {
 
         return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
@@ -181,6 +243,10 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * @param bBoxPerPage initial empty BoundingBox
+     * @return The union of all BoundingBoxes of the TextBlock of this node
+     */
     private Map<PageNode, Rectangle2D> getBBoxFromTerminalTextBlock(Map<PageNode, Rectangle2D> bBoxPerPage) {
 
         Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java

@@ -8,7 +8,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@@ -25,7 +24,7 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableCellNode extends PageElement implements SemanticNode {
+public class TableCellNode implements SemanticNode {
 
     List<Integer> tocId;
     int row;
@@ -60,6 +59,10 @@ public class TableCellNode extends PageElement implements SemanticNode {
     @Override
     public TextBlock buildTextBlock() {
 
+        if (terminal) {
+            return terminalTextBlock;
+        }
+
         if (textBlock == null) {
             textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
         }
@@ -76,7 +79,7 @@ public class TableCellNode extends PageElement implements SemanticNode {
 
     public boolean hasHeader(String headerString) {
 
-        return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().contains(headerString));
+        return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().strip().equals(headerString));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java

@@ -5,7 +5,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@@ -22,7 +21,7 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableNode extends PageElement implements SemanticNode {
+public class TableNode implements SemanticNode {
 
     List<Integer> tocId;
     TableOfContents tableOfContents;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
-import java.util.LinkedList;
 import java.util.List;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
@@ -121,11 +120,7 @@ public class AtomicTextBlock implements TextBlock {
                 .map(RectangleTransformations::rectangleUnion)
                 .toList();
 
-        List<EntityPosition> entityPositions = new LinkedList<>();
+        return List.of(EntityPosition.builder().rectanglePerLine(positionsPerLine).pageNode(page).build());
-        for (Rectangle2D position : positionsPerLine) {
-            entityPositions.add(EntityPosition.builder().position(position).pageNode(page).build());
-        }
-        return entityPositions;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java

@@ -1,14 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
 
 import static java.lang.String.format;
+import static java.util.stream.Collectors.flatMapping;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
 
 import java.awt.geom.Rectangle2D;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.function.Supplier;
+import java.util.Map;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 
 import lombok.AccessLevel;
 import lombok.Data;
@@ -16,7 +20,7 @@ import lombok.experimental.FieldDefaults;
 
 @Data
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTextBlock> {
+public class ConcatenatedTextBlock implements TextBlock {
 
     List<AtomicTextBlock> atomicTextBlocks;
     String searchText;
@@ -68,7 +72,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
     public String getSearchText() {
 
         if (searchText == null) {
-            StringBuilder sb  = new StringBuilder();
+            StringBuilder sb = new StringBuilder();
             getAtomicTextBlocks().forEach(atb -> sb.append(atb.getSearchText()));
             searchText = sb.toString();
         }
@@ -96,6 +100,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
         return getAtomicTextBlockByStringIndex(fromIndex).getPreviousLinebreak(fromIndex);
     }
 
+
     @Override
     public List<Integer> getLineBreaks() {
 
@@ -152,14 +157,20 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
         AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
         positions.addAll(lastTextBlock.getEntityPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
 
-        return positions;
+        return mergeEntityPositionsWithSamePageNode(positions);
     }
 
 
-    @Override
+    private List<EntityPosition> mergeEntityPositionsWithSamePageNode(List<EntityPosition> positions) {
-    public ConcatenatedTextBlock get() {
+
+        Map<PageNode, List<Rectangle2D>> entityPositionsPerPage = positions.stream().collect(//
+                groupingBy(EntityPosition::getPageNode, //
+                        flatMapping(entityPosition -> entityPosition.getRectanglePerLine().stream(), toList())));
+
+        return entityPositionsPerPage.entrySet().stream()//
+                .map(entry -> EntityPosition.builder().pageNode(entry.getKey()).rectanglePerLine(entry.getValue()).build())//
+                .toList();
 
-        return this;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java

@@ -99,7 +99,7 @@ public interface TextBlock extends CharSequence {
     @Override
     default int length() {
 
-        return getSearchText().length();
+        return getBoundary().length();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java

@@ -13,11 +13,10 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public class TextBlockCollector implements Collector<TextBlock, ConcatenatedTextBlock, TextBlock> {
 
-
     @Override
     public Supplier<ConcatenatedTextBlock> supplier() {
 
-        return new ConcatenatedTextBlock(Collections.emptyList());
+        return () -> new ConcatenatedTextBlock(Collections.emptyList());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java

@@ -1,7 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.document.services;
 
-import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -9,7 +10,6 @@ import java.util.stream.Collectors;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
-import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
@@ -30,64 +30,79 @@ public class EntityCreationService {
     private final EntityTextEnrichmentService entityEnrichmentService;
 
 
-    public Set<EntityNode> createEntitiesByLineAfterString(String string, SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) {
+    public Set<EntityNode> createEntitiesByBetweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
+
+        return Collections.emptySet();
+    }
+
+
+    public Set<EntityNode> createEntitiesBySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
+
+        return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
+                .stream()
+                .filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary))
+                .map(bounds -> createEntityByBoundary(bounds, type, entityType, node))
+                .collect(Collectors.toUnmodifiableSet());
+    }
+
+
+    public Set<EntityNode> createEntitiesByLineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.buildTextBlock();
         SearchImplementation searchImplementation = new SearchImplementation(List.of(string), true);
         List<Boundary> boundaries = searchImplementation.getBoundaries(textBlock, node.getBoundary());
         return boundaries.stream()
                 .map(boundary -> toLineAfterBoundary(textBlock, boundary))
-                .map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph))
+                .map(boundary -> createEntityByBoundary(boundary, type, entityType, node))
                 .collect(Collectors.toUnmodifiableSet());
     }
 
 
-    public Set<EntityNode> createEntitiesByRegex(String regexPattern, String type, EntityType entityType, DocumentGraph documentGraph) {
+    public Set<EntityNode> createEntitiesByRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
 
-        List<Boundary> boundaries = RegexMatcher.findBoundaries(regexPattern, documentGraph.buildTextBlock());
+        List<Boundary> boundaries = RegexMatcher.findBoundaries(regexPattern, node.buildTextBlock());
-        return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph)).collect(Collectors.toUnmodifiableSet());
+        return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
     }
 
 
-    public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, DocumentGraph documentGraph) {
+    public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType) {
+
+        Boundary boundary = node.buildTextBlock().getBoundary();
+        Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
+        return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node);
+    }
+
+
+    public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
 
         EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType);
-        addEntityToGraph(entity, documentGraph);
+        addEntityToGraph(entity, node.getTableOfContents());
         return entity;
     }
 
 
-    public EntityNode createEntityByEntity(EntityNode entity, String type, EntityType entityType, DocumentGraph documentGraph) {
+    public EntityNode createMergedEntity(List<EntityNode> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
 
-        return createEntityByBoundary(entity.getBoundary(), type, entityType, documentGraph);
+        if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
-    }
+            throw new IllegalArgumentException("Provided entities can not be merged!");
-
-
-    public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) {
-
-        Boundary boundary = node.buildTextBlock().getBoundary();
-        Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
-        return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, documentGraph);
-    }
-
-
-    public Set<EntityNode> createEntitiesByString(String entityName, int startOffset, String type, EntityType entityType, DocumentGraph documentGraph) {
-
-        int start = documentGraph.buildTextBlock().indexOf(entityName, startOffset);
-        if (start == -1) {
-            throw new NotFoundException(format("Entity %s could no be found!", entityName));
         }
-        EntityNode entity = EntityNode.initialEntityNode(new Boundary(start, start + entityName.length()), type, entityType);
+        EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType);
-        addEntityToGraph(entity, documentGraph);
+        mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction));
-        return Set.of(entity);
+        mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
+        if (mergedEntity.isRedaction()) {
+            mergedEntity.setRedactionReason(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getRedactionReason).findFirst().orElse(""));
+            mergedEntity.setMatchedRule(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getMatchedRule).findFirst().orElse(-1));
+            mergedEntity.setLegalBasis(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getLegalBasis).findFirst().orElse(""));
+        }
+        addEntityToGraph(mergedEntity, node.getTableOfContents());
+        return mergedEntity;
     }
 
 
-    public void addEntityToGraph(EntityNode entity, DocumentGraph documentGraph) {
+    public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
 
         try {
-            SemanticNode containingNode = documentGraph.getTableOfContents()
+            SemanticNode containingNode = tableOfContents.getEntries()
-                    .getEntries()
                     .stream()
                     .map(TableOfContents.Entry::node)
                     .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
@@ -103,7 +118,7 @@ public class EntityCreationService {
             addToNodeEntitySets(entity);
 
         } catch (NotFoundException e) {
-            entityEnrichmentService.enrichEntity(entity, documentGraph.getTextBlock());
+            entityEnrichmentService.enrichEntity(entity, tableOfContents.buildTextBlock());
             log.warn("Entity \"{}\" with {} is in between two main sections and will be removed!", entity.getValue(), entity.getBoundary());
             entity.removeFromGraph();
         }
@@ -123,6 +138,27 @@ public class EntityCreationService {
         entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
     }
 
+
+    private static boolean allEntitiesIntersectAndHaveSameTypes(List<EntityNode> entitiesToMerge) {
+
+        if (entitiesToMerge.isEmpty()) {
+            return true;
+        }
+        EntityNode previousEntity = entitiesToMerge.get(0);
+        for (EntityNode entityNode : entitiesToMerge.subList(1, entitiesToMerge.size())) {
+            boolean typeMatches = entityNode.getType().equals(previousEntity.getType());
+            boolean entityTypeMatches = entityNode.getEntityType().equals(previousEntity.getEntityType());
+            boolean intersects = entityNode.intersects(previousEntity);
+            if (typeMatches && entityTypeMatches && intersects) {
+                previousEntity = entityNode;
+                continue;
+            }
+            return false;
+        }
+        return true;
+    }
+
+
     private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) {
 
         return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java

@@ -0,0 +1,32 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import java.util.Collection;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
+
+public class EntityFieldSetter {
+
+    public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
+
+        if (entityNode.getMatchedRule() == -1 && matchedRule != -1) {
+            entityNode.setMatchedRule(matchedRule);
+        }
+        if (entityNode.getRedactionReason().equals("") && redactionReason != null) {
+            entityNode.setRedactionReason(redactionReason);
+        }
+        if (entityNode.getLegalBasis().equals("") && legalBasis != null) {
+            entityNode.setLegalBasis(legalBasis);
+        }
+        if (engine != null) {
+            entityNode.addEngine(engine);
+        }
+    }
+
+
+    public static void setFields(Collection<EntityNode> entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
+
+        entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java

@@ -1,19 +1,29 @@
 package com.iqser.red.service.redaction.v1.server.document.services;
 
+import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
+
 import java.util.LinkedList;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 
 public class RegexMatcher {
 
-    public static boolean anyMatch(CharSequence searchText, String regexPattern) {
+    public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
 
         Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
-        return pattern.matcher(searchText).find();
+        return pattern.matcher(charSequence).find();
+    }
+
+
+    public static boolean anyMatch(TextBlock textBlock, String regexPattern) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+        return pattern.matcher(textBlock).region(textBlock.getBoundary().start(), textBlock.getBoundary().end()).find();
     }
 
 
@@ -24,15 +34,16 @@ public class RegexMatcher {
         return new Boundary(matcher.start(), matcher.end());
     }
 
-    public static List<Boundary> findBoundaries(String regexPattern, CharSequence searchText) {
+
+    public static List<Boundary> findBoundaries(String regexPattern, TextBlock textBlock) {
 
         Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
-        Matcher matcher = pattern.matcher(searchText);
+        Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary()));
         List<Boundary> boundaries = new LinkedList<>();
         while (matcher.find()) {
-            boundaries.add(new Boundary(matcher.start(), matcher.end()));
+            boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start()));
         }
-        return boundaries;
+        return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java

@@ -1,11 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import static java.lang.String.format;
+
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
+
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+
 import lombok.Data;
 import lombok.Getter;
 
@@ -50,8 +55,11 @@ public class Dictionary {
 
 
     public DictionaryModel getType(String type) {
+
         DictionaryModel model = localAccessMap.get(type);
-        if (model == null) throw new NotFoundException("Type: " + type + " is not found");
+        if (model == null) {
+            throw new NotFoundException("Type: " + type + " is not found");
+        }
         return model;
     }
 
@@ -75,4 +83,23 @@ public class Dictionary {
         return false;
     }
 
+
+    public void addLocalDictionaryEntry(String type, String value, boolean alsoAddLastname) {
+
+        if (localAccessMap.get(type) == null) {
+            throw new IllegalArgumentException(format("DictionaryModel of type %s does not exist", type));
+        }
+        if (localAccessMap.get(type).getLocalEntries() == null) {
+            throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type));
+        }
+        if (StringUtils.isEmpty(value) && value.length() < 3) {
+            throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
+        }
+        localAccessMap.get(type).getLocalEntries().add(value);
+        if (alsoAddLastname) {
+            String lastname = value.split(" ")[0];
+            localAccessMap.get(type).getLocalEntries().add(lastname);
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -104,10 +104,12 @@ public class DroolsExecutionService {
 
     }
 
+
     @Timed("redactmanager_executeRules")
     public Section executeRules(KieContainer kieContainer, Section section) {
 
         KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("section", section);
         kieSession.insert(section);
         kieSession.fireAllRules();
         kieSession.dispose();
@@ -116,7 +118,9 @@ public class DroolsExecutionService {
 
     }
 
+
     public List<Entity> getEntities(KieSession ks) {
+
         List<Entity> entities = new LinkedList<>();
         QueryResults entitiesResult = ks.getQueryResults("getEntities");
         for (QueryResultsRow resultsRow : entitiesResult) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -1,7 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle;
+
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -14,7 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@@ -35,19 +36,73 @@ public class RedactionLogCreatorService {
     private final DictionaryService dictionaryService;
 
 
-    public List<RedactionLogEntry> createRedactionLog(Map<PageNode, Set<EntityNode>> entityNodesPerPageNode, int numberOfPages, String dossierTemplateId) {
+    public List<RedactionLogEntry> createRedactionLog(Set<EntityNode> entityNodes, String dossierTemplateId) {
 
         List<RedactionLogEntry> entries = new ArrayList<>();
-        entityNodesPerPageNode.forEach(((pageNode, entityNodes) -> entries.addAll(toRedactionLogEntries(pageNode, entityNodes, dossierTemplateId))));
+        Set<String> processedIds = new HashSet<>();
-
+        entityNodes.forEach((entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId))));
 
         return entries;
     }
 
 
-    private Set<RedactionLogEntry> toRedactionLogEntries(PageNode pageNode, Set<EntityNode> entityNodes, String dossierTemplateId) {
+    private List<RedactionLogEntry> toRedactionLogEntries(EntityNode entityNode, Set<String> processedIds, String dossierTemplateId) {
 
-        return Collections.emptySet();
+        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
+
+        // Duplicates can exist due table extraction columns over multiple rows.
+
+        for (EntityPosition entityPosition : entityNode.getEntityPositions()) {
+
+            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entityNode, dossierTemplateId);
+
+            if (processedIds.contains(entityPosition.getId())) {
+                continue;
+            }
+
+            processedIds.add(entityPosition.getId());
+
+            redactionLogEntry.setId(entityPosition.getId());
+
+            List<Rectangle> rectanglesPerLine = entityPosition.getRectanglePerLine()
+                    .stream()
+                    .map(rectangle2D -> toRectangle(rectangle2D, entityPosition.getPageNode().getNumber()))
+                    .toList();
+
+            redactionLogEntry.setPositions(rectanglesPerLine);
+        }
+
+        return redactionLogEntities;
+    }
+
+
+    private RedactionLogEntry createRedactionLogEntry(EntityNode entity, String dossierTemplateId) {
+
+        Set<String> referenceIds = new HashSet<>();
+        entity.getReferences().forEach(ref -> ref.getEntityPositions().forEach(pos -> referenceIds.add(pos.getId())));
+
+        return RedactionLogEntry.builder()
+                .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
+                .reason(entity.getRedactionReason())
+                .legalBasis(entity.getLegalBasis())
+                .value(entity.getValue())
+                .type(entity.getType())
+                .redacted(entity.isRedaction())
+                .isHint(isHint(entity.getType(), dossierTemplateId))
+                .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
+                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
+                .section(entity.getDeepestFullyContainingNode().getTocId().toString())
+                .sectionNumber(entity.getDeepestFullyContainingNode().getTocId().get(0))
+                .matchedRule(entity.getMatchedRule())
+                .isDictionaryEntry(entity.isDictionaryEntry())
+                .textAfter(entity.getTextAfter().toString())
+                .textBefore(entity.getTextBefore().toString())
+                .startOffset(entity.getBoundary().start())
+                .endOffset(entity.getBoundary().end())
+                .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
+                .engines(entity.getEngines())
+                .reference(referenceIds)
+                .build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -124,7 +124,7 @@ public final class EntitySearchUtils {
                     orderedEntities.get(i).setPositionSequences(toAdd);
 
                 } catch (Exception e) {
-                    log.warn("Mismatch between EntityPositionSequence and found Entity!");
+                    //log.warn("Mismatch between EntityPositionSequence and found Entity!");
                 }
             }
         }
@@ -209,9 +209,7 @@ public final class EntitySearchUtils {
                         .length() && inner.getStart() >= outer.getStart() && inner.getEnd() <= outer.getEnd() && outer != inner && outer.getSectionNumber() == inner.getSectionNumber()) {
                     if (outer.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                         wordsToRemove.add(outer);
-                    } else if (!(
+                    } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && outer.getEntityType() == EntityType.ENTITY || inner.getEntityType() == EntityType.ENTITY && outer.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
-                            inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && outer.getEntityType() == EntityType.ENTITY ||
-                            inner.getEntityType() == EntityType.ENTITY && outer.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
                         if (inner.isResized()) {
                             wordsToRemove.add(outer);
                         } else {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java

@@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
 import java.util.Set;
 import java.util.regex.Pattern;
 
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
@@ -28,4 +30,23 @@ public final class SeparatorUtils {
         return intValue >= 12288 && intValue <= 12336 || japaneseAltPunctuationMarks.contains(intValue);
     }
 
+
+
+    public static boolean validateBoundaryIsSurroundedBySeparators(CharSequence charSequence, Boundary boundary) {
+
+        return validateStart(charSequence, boundary) && validateEnd(charSequence, boundary);
+    }
+
+
+    private static boolean validateEnd(CharSequence charSequence, Boundary boundary) {
+
+        return boundary.end() == charSequence.length() || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end())) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end() - 1));
+    }
+
+
+    private static boolean validateStart(CharSequence charSequence, Boundary boundary) {
+
+        return boundary.start() == 0 || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start()));
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
 import static com.iqser.red.service.redaction.v1.server.utils.PdfDraw.drawRectangle2DList;
 
 import java.awt.Color;
@@ -18,10 +19,9 @@ import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
 import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
@@ -29,11 +29,11 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.utils.PdfDraw;
 
@@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
     @SneakyThrows
     public void testDroolsOnDocumentGraph() {
 
-        String filename = "files/new/crafted document";
+        String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
 
         prepareStorage(filename + ".pdf");
         ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
@@ -76,31 +76,39 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
         dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
 
+        long dictionarySearchStart = System.currentTimeMillis();
         List<EntityNode> foundEntities = new LinkedList<>();
-        for (var model : dictionary.getDictionaryModels()) {
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
             findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
             findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
             findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
-            foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document));
         }
+        System.out.printf("Dictionary search took %d ms and found %d entities\n", System.currentTimeMillis() - dictionarySearchStart, foundEntities.size());
+
+        long graphInsertionStart = System.currentTimeMillis();
+        foundEntities.forEach(entity -> entity.setDossierDictionaryEntry(true));
+        foundEntities.forEach(entity -> entity.setDictionaryEntry(true));
+        foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document.getTableOfContents()));
+        System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart);
 
         KieSession kieSession = kieContainer.newKieSession();
-        entityCreationService.createEntitiesByString("Michael N.", 0, "CBI_author", EntityType.FALSE_POSITIVE, document)//
-                .forEach(kieSession::insert);
         kieSession.setGlobal("document", document);
         kieSession.setGlobal("entityCreationService", entityCreationService);
+        kieSession.setGlobal("dictionary", dictionary);
 
+        long serializationStart = System.currentTimeMillis();
         document.getEntities().forEach(kieSession::insert);
         document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node()));
         document.getPages().forEach(kieSession::insert);
+        System.out.printf("Object serialization and kieSession insertion took %d ms\n", System.currentTimeMillis() - serializationStart);
 
-        long start = System.currentTimeMillis();
+        long dictionaryAddsStart = System.currentTimeMillis();
         kieSession.insert(FileAttribute.builder().label("Vertebrate Study").value("Yes").build());
-
+        kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
         kieSession.fireAllRules();
-        System.out.printf("Firing all rules took %d ms\n", System.currentTimeMillis() - start);
+        System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - dictionaryAddsStart);
-
+        System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart);
-        //List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog();
+        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document.getEntities(), TEST_DOSSIER_TEMPLATE_ID);
         drawAllEntities(filename, fileResource, document);
     }
 
@@ -181,7 +189,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
 
             var insertStart = System.currentTimeMillis();
             DocumentGraph finalDocument = document;
-            foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument));
+            foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument.getTableOfContents()));
             var insertTime = System.currentTimeMillis() - insertStart;
             totalInsertTime += insertTime;
 
@@ -210,7 +218,8 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
                         .filter(entityNode -> !entityNode.isRemoved())
                         .filter(EntityNode::isRedaction)
                         .flatMap(entityNode -> entityNode.getEntityPositions().stream())
-                        .map(EntityPosition::getPosition)
+                        .filter(entityPosition -> entityPosition.getPageNode().equals(page))
+                        .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
                         .toList();
 
                 PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLACK).stroke(true).build();
@@ -221,9 +230,10 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
                 List<Rectangle2D> entityPositionsOnPage = page.getEntities()
                         .stream()
                         .filter(entityNode -> !entityNode.isRemoved())
-                        .filter(e -> !e.isRedaction())
+                        .filter(entityNode -> !entityNode.isRedaction())
                         .flatMap(entityNode -> entityNode.getEntityPositions().stream())
-                        .map(EntityPosition::getPosition)
+                        .filter(entityPosition -> entityPosition.getPageNode().equals(page))
+                        .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
                         .toList();
 
                 PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLUE).stroke(true).build();
@@ -245,16 +255,9 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
         TextBlock textBlock = documentGraph.getTextBlock();
         searchImplementation.getBoundaries(textBlock, textBlock.getBoundary())
                 .stream()
-                .filter(boundary -> validateSeparators(textBlock, boundary))
+                .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
                 .map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType))
                 .forEach(foundEntities::add);
     }
 
-
-    private static boolean validateSeparators(TextBlock textBlock, Boundary boundary) {
-
-        return (boundary.start() == 0 || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start()))) && (boundary.end() == textBlock.length() || SeparatorUtils.isSeparator(
-                textBlock.charAt(boundary.end())) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1)));
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java

@@ -7,8 +7,8 @@ import org.springframework.core.io.ClassPathResource;
 import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
 import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentDataMapper;
-import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 
@@ -46,11 +46,6 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
 
         assert document.toString().equals(newDocumentGraph.toString());
         assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString());
-        for (int pageIndex = 1; pageIndex < document.getNumberOfPages(); pageIndex++) {
-            var page = document.getPages().get(pageIndex);
-            var newPage = document.getPages().get(pageIndex);
-            assert page.toString().equals(newPage.toString());
-        }
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java

@@ -14,16 +14,16 @@ import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@@ -56,7 +56,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
 
         assertEquals("Expand to Hint ", entityNode.getTextBefore());
         assertEquals("’s Donut ←", entityNode.getTextAfter());
@@ -80,7 +80,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
 
         assertEquals("", entityNode.getTextBefore());
         assertEquals(" Purity Hint", entityNode.getTextAfter());
@@ -103,7 +103,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
 
         assertEquals("", entityNode.getTextBefore());
         assertEquals("", entityNode.getTextAfter());
@@ -189,7 +189,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
 
         assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore());
         assertEquals(", Group 9;", entityNode.getTextAfter());
@@ -238,7 +238,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
 
         assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore());
         assertEquals(" of metabolite of", entityNode.getTextAfter());
@@ -293,8 +293,8 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
-        PageNode pageNode = documentGraph.getPages().get(pageNumber - 1);
+        PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
 
         assertEquals(entityNode.getValue(), searchTerm);
         assertTrue(pageNode.getEntities().contains(entityNode));

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl

@@ -2,6 +2,7 @@ package drools
 
 import static java.lang.String.format;
 import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields;
 
 
 import java.util.List;
@@ -17,9 +18,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
 import java.util.Set
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
 import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 
 global DocumentGraph document
 global EntityCreationService entityCreationService
+global Dictionary dictionary
+
+// --------------------------------------- CBI rules -------------------------------------------------------------------
 
 rule "1: Redact CBI Authors (Non vertebrate study)"
     no-loop true
@@ -29,14 +34,7 @@ rule "1: Redact CBI Authors (Non vertebrate study)"
         $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
     then
         $entity.setRedaction(true);
-        if ($entity.getMatchedRule() == -1) {
+        setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
-            $entity.setMatchedRule(1);
-        }
-        if ($entity.getRedactionReason().equals("")) {
-            $entity.setRedactionReason("Author found");
-        }
-        $entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
-        $entity.addEngine(Engine.RULE);
         update($entity)
     end
 
@@ -48,14 +46,7 @@ rule "2: Redact CBI Authors (Vertebrate study)"
         $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
     then
         $entity.setRedaction(true);
-        if ($entity.getMatchedRule() == -1) {
+        setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
-            $entity.setMatchedRule(2);
-        }
-        if ($entity.getRedactionReason().equals("")) {
-            $entity.setRedactionReason("Author found");
-        }
-        $entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
-        $entity.addEngine(Engine.RULE);
         update($entity)
     end
 
@@ -65,15 +56,9 @@ rule "3: Don't redact CBI Address (Non vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
         $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
-        $recommendationEntity: EntityNode(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
     then
-        $entity.setRedaction(false);
+        setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
-        $entity.setMatchedRule(3);
-        $entity.setRedactionReason("Address found for non vertebrate study");
-        $entity.addEngine(Engine.RULE);
         update($entity)
-        $recommendationEntity.removeFromGraph();
-        delete($recommendationEntity);
     end
 
 rule "4: Redact CBI Address (Vertebrate study)"
@@ -84,43 +69,43 @@ rule "4: Redact CBI Address (Vertebrate study)"
         $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
     then
         $entity.setRedaction(true);
-        $entity.setMatchedRule(4);
+        setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
-        $entity.setRedactionReason("Address found");
-        $entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
-        $entity.addEngine(Engine.RULE);
         update($entity)
     end
 
 rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
 
     when
-        $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
+        $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
     then
-        EntityNode entity = entityCreationService.createEntityByEntity($entity, "CBI_author", EntityType.FALSE_POSITIVE, document);
+        EntityNode entity = entityCreationService.createEntityByBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
-        entity.setMatchedRule(5);
+        setFields($entity, 5, "Genitive Author", null, Engine.RULE);
         insert(entity);
     end
 
 
-rule "6: Create Entity from Author(s) cells in Tables with Author(s) header"
+rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
 
     when
-        authorCell: TableCellNode(header == false, (hasHeader("Author(s)") || hasHeader("Author")))
+        authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
     then
-        EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY, document);
+        EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
-        entity.setMatchedRule(6);
+        setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
-        entity.setRedactionReason("Header \"Author(s)\" found");
         insert(entity);
+        dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
     end
 
 rule "7: Add CBI_author with \"et al.\" Regex"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
 
     when
+         $section: SectionNode(containsString("et al."))
     then
-         Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, document);
+         Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
-         entities.forEach(entity -> entity.setMatchedRule(7));
+         setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
-         entities.forEach(entity -> entity.setRedactionReason("Found by \"et al.\" regex"));
          entities.forEach(entity -> insert(entity));
+         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
     end
 
 rule "8: Add recommendation for Addresses in Test Organism sections"
@@ -129,9 +114,8 @@ rule "8: Add recommendation for Addresses in Test Organism sections"
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $section: SectionNode(containsString("Species") && containsString("Source"))
     then
-        Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source", $section, "CBI_address", EntityType.RECOMMENDATION, document);
+        Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section);
-        entities.forEach(entity -> entity.setRedactionReason("Line after \"Source\" in Test Organism Section"));
+        setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
-        entities.forEach(entity -> entity.setMatchedRule(8));
         entities.forEach(entity -> insert(entity));
     end
 
@@ -142,9 +126,46 @@ rule "9: Add recommendation for Addresses in Test Animals sections"
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
         $section: SectionNode(containsString("Species:") && containsString("Source:"))
     then
-        Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source:", $section, "CBI_address", EntityType.RECOMMENDATION, document);
+        Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
-        entities.forEach(entity -> entity.setRedactionReason("Line after \"Source:\" in Test Organism Section"));
+        setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
-        entities.forEach(entity -> entity.setMatchedRule(9));
         entities.forEach(entity -> insert(entity));
     end
 
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+
+rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
+    no-loop true
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
+        $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
+    then
+        $entity.setRedaction(true);
+        setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
+    end
+
+
+
+rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
+    no-loop true
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
+    then
+        $entity.setRedaction(true);
+        setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
+    end
+
+rule "12: Redact Emails by RegEx (Non vertebrate study)"
+
+    when
+         $section: SectionNode(containsString("@"))
+    then
+         Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
+         setFields(entities, 12, "Found by email regex", null, Engine.RULE);
+         entities.forEach(entity -> insert(entity));
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl

@@ -8,75 +8,88 @@ import java.util.List;
 import java.util.LinkedList;
 import java.util.HashSet;
 
+import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.document.graph.*
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import java.util.Set;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 
 global DocumentGraph document
+global EntityCreationService entityCreationService
+global Dictionary dictionary
 
 
-rule "merge contained Entities of same type"
+rule "merge intersecting Entities of same type"
     salience 100
+
     when
-        outer: EntityNode($type: type, $entityType: entityType)
+        $first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
-        inner: EntityNode(this != outer, containedBy(outer), type == $type, entityType == $entityType)
+        $second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        System.out.printf("removed entity %s contained by %s\n", inner, outer);
+        $first.removeFromGraph();
-        outer.addEngines(inner.getEngines());
+        $second.removeFromGraph();
-        inner.removeFromGraph();
+        EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document);
-        delete(inner);
+        insert(mergedEntity);
+        delete($first);
+        delete($second);
     end
 
 rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 100
+
     when
         $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
-        entity: EntityNode(this != $falsePositive, containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY)
+        entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        System.out.printf("removed entity %s marked false positive by %s\n", entity, $falsePositive);
         entity.removeFromGraph();
         delete(entity);
     end
 
-/*
+rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
-rule "If same type, remove inner"
     salience 100
+
     when
-        $containedEntity: ContainedEntity($toRemove: inner, outer.getType() == inner.getType())
+        $falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
+        $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-         $toRemove.removeFromGraph();
+        $recommendation.removeFromGraph();
-         delete($toRemove);
+        delete($recommendation);
-         delete($containedEntity);
     end
 
-rule "If outer Recommended and inner Entity, remove outer"
+rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
-    salience 99
+    salience 100
+
     when
-        $containedEntity: ContainedEntity($toRemove: outer, outer.getType() == EntityType.RECOMMENDATION, inner.getType() == EntityType.ENTITY)
+        $entity: EntityNode($type: type, entityType == EntityType.ENTITY)
+        $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $toRemove.removeFromGraph();
+        $recommendation.removeFromGraph();
-        delete($toRemove);
+        delete($recommendation);
-        delete($containedEntity);
     end
 
-rule "Inner not False Recommendation and outer not Entity"
+rule "remove Entity of lower rank, when equal boundaries and entityType"
-    salience 99
+    salience 100
+
     when
-        $containedEntity: ContainedEntity($toRemove: outer, (outer.getType() != EntityType.ENTITY || inner.getType() != EntityType.FALSE_RECOMMENDATION))
+        $higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
+        $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
     then
-        $toRemove.removeFromGraph();
+        $lowerRank.removeFromGraph();
-        delete($toRemove);
+        delete($lowerRank);
-        delete($containedEntity);
-    end
-rule "Remove leftover ContainedEntities"
-    salience 0
-    when
-        $toRemove: ContainedEntity()
-    then
-        delete($toRemove);
     end
 
-*/
+rule "run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+
+    when
+        DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
+    then
+        Set<EntityNode> entityNodes = entityCreationService.createEntitiesBySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
+        System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
+        entityNodes.forEach(entityNode -> insert(entityNode));
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -0,0 +1,367 @@
+package drools
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+
+global Section section
+
+
+// --------------------------------------- AI rules -------------------------------------------------------------------
+
+rule "0: Add CBI_author from ai"
+    when
+        Section(aiMatchesType("CBI_author"))
+    then
+        section.addAiEntities("CBI_author", "CBI_author");
+    end
+
+rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
+    when
+        Section(aiMatchesType("ORG"))
+    then
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
+    when
+        Section(aiMatchesType("STREET"))
+    then
+        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
+    when
+        Section(aiMatchesType("CITY"))
+    then
+        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
+    end
+
+
+// --------------------------------------- CBI rules -------------------------------------------------------------------
+
+rule "1: Redact CBI Authors (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "2: Redact CBI Authors (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "3: Redact not CBI Address (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
+        section.ignoreRecommendations("CBI_address");
+    end
+
+rule "4: Redact CBI Address (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "5: Do not redact genitive CBI_author"
+    when
+        Section(matchesType("CBI_author"))
+    then
+        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
+    end
+
+
+rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "16: Add recommendation for Addresses in Test Organism sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
+    then
+        section.recommendLineAfter("Source:", "CBI_address");
+    end
+
+rule "17: Add recommendation for Addresses in Test Animals sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
+    then
+        section.recommendLineAfter("Source", "CBI_address");
+    end
+
+
+rule "18: Do not redact Names and Addresses if Published Information found"
+    when
+        Section(matchesType("published_information"))
+    then
+        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
+        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
+    end
+
+
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+
+rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "21: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "22: Redact Emails by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "23: Redact contact information (Non vertebrate study)"
+    when
+       Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+            || text.contains("Contact:")
+            || text.contains("Alternative contact:")
+            || (text.contains("No:") && text.contains("Fax"))
+            || (text.contains("Contact:") && text.contains("Tel.:"))
+            || text.contains("European contact:")
+        ))
+    then
+        section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+     end
+
+rule "24: Redact contact information (Vertebrate study)"
+    when
+       Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+            || text.contains("Contact:")
+            || text.contains("Alternative contact:")
+            || (text.contains("No:") && text.contains("Fax"))
+            || (text.contains("Contact:") && text.contains("Tel.:"))
+            || text.contains("European contact:")
+        ))
+    then
+        section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+     end
+
+rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
+               text.contains("Contact")
+            || text.contains("Telephone")
+            || text.contains("Phone")
+            || text.contains("Fax")
+            || text.contains("Tel")
+            || text.contains("Ter")
+            || text.contains("Mobile")
+            || text.contains("Fel")
+            || text.contains("Fer")
+        ))
+    then
+        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
+               text.contains("Contact")
+            || text.contains("Telephone")
+            || text.contains("Phone")
+            || text.contains("Fax")
+            || text.contains("Tel")
+            || text.contains("Ter")
+            || text.contains("Mobile")
+            || text.contains("Fel")
+            || text.contains("Fer")
+        ))
+    then
+        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+rule "27: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("COMPLETION DATE:")
+            && !searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "28: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("COMPLETION DATE:")
+            && !searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "29: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "30: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("PERFORMING LABORATORY:")
+        )
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
+    end
+
+rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("PERFORMING LABORATORY:"))
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+rule "33: Redact study director abbreviation"
+    when
+        Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
+    then
+        section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+     end
+
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "34: Purity Hint"
+    when
+        Section(searchText.toLowerCase().contains("purity"))
+    then
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+    end
+
+
+
+rule "35: Redact signatures (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "36: Redact signatures (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "43: Redact Logos (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
+    then
+        section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end