RED-6368: Include Tables and Images into Prototype Document Structure

*rebased to master and included tenant capabilities
*added separator validation to entity search
*added EntityPosition class to bridge the gap to the redactionLogEntry

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.document.services;
+package com.iqser.red.service.redaction.v1.server.document.data.mapper;
 
 import java.awt.geom.Rectangle2D;
 import java.util.List;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.document.services;
+package com.iqser.red.service.redaction.v1.server.document.data.mapper;
 
 import static java.lang.Math.toIntExact;
 import static java.lang.String.format;
@@ -71,7 +71,7 @@ public class DocumentGraphMapper {
 
     private void buildSection(TableOfContentsData.EntryData entryData, List<Integer> currentTocId, Context context) {
 
-        SectionNode section = SectionNode.builder().entities(new HashSet<>()).pages(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
+        SectionNode section = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
 
         context.sections().add(section);
 
@@ -90,7 +90,7 @@ public class DocumentGraphMapper {
         Set<PageNode> pages = Arrays.stream(entryData.pages()).map(pageNumber -> getPage(pageNumber, context)).collect(Collectors.toSet());
 
         SectionNode parentSection = (SectionNode) context.tableOfContents().getEntryById(currentTocId).node();
-        ParagraphNode paragraph = ParagraphNode.builder().pages(pages).build();
+        ParagraphNode paragraph = ParagraphNode.builder().build();
         TextBlock textBlock = toTextBlock(entryData.atomicTextBlocks(), context, paragraph);
 
         paragraph.setTerminalTextBlock(textBlock);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java

@@ -2,6 +2,9 @@ package com.iqser.red.service.redaction.v1.server.document.graph;
 
 import static java.lang.String.format;
 
+import java.util.LinkedList;
+import java.util.List;
+
 import lombok.Setter;
 
 @Setter
@@ -47,14 +50,14 @@ public class Boundary implements Comparable<Boundary> {
 
     public boolean containedBy(Boundary boundary) {
 
-        return boundary.start() <= start && end <= boundary.end();
+        return boundary.contains(this);
     }
 
 
     public boolean contains(int start, int end) {
 
         if (start > end) {
-            throw new UnsupportedOperationException("start > end");
+            throw new IllegalArgumentException(format("start: %d > end: %d", start, end));
         }
         return this.start <= start && end <= this.end;
     }
@@ -63,7 +66,7 @@ public class Boundary implements Comparable<Boundary> {
     public boolean containedBy(int start, int end) {
 
         if (start > end) {
-            throw new UnsupportedOperationException("start > end");
+            throw new IllegalArgumentException(format("start: %d > end: %d", start, end));
         }
         return start <= this.start && this.end <= end;
     }
@@ -81,6 +84,22 @@ public class Boundary implements Comparable<Boundary> {
     }
 
 
+    public List<Boundary> split(List<Integer> splitIndices) {
+
+        if(splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
+            throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this));
+        }
+        List<Boundary> splitBoundaries = new LinkedList<>();
+        int previousIndex = start;
+        for (int splitIndex : splitIndices) {
+            splitBoundaries.add(new Boundary(previousIndex, splitIndex));
+            previousIndex = splitIndex;
+        }
+        splitBoundaries.add(new Boundary(previousIndex, end));
+        return splitBoundaries;
+    }
+
+
     @Override
     public String toString() {
 
@@ -100,4 +119,16 @@ public class Boundary implements Comparable<Boundary> {
         return 0;
     }
 
+    @Override
+    public int hashCode() {
+
+        return toString().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object object) {
+
+        return hashCode() == object.hashCode();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java

@@ -7,7 +7,7 @@ import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java

@@ -0,0 +1,166 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.hash.Hashing;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class EntityNode {
+
+    public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) {
+
+        return EntityNode.builder()
+                .type(type)
+                .entityType(entityType)
+                .boundary(boundary)
+                .redaction(false)
+                .falsePositive(false)
+                .removed(false)
+                .ignored(false)
+                .resized(false)
+                .skipRemoveEntitiesContainedInLarger(false)
+                .dictionaryEntry(false)
+                .dossierDictionaryEntry(false)
+                .engines(new HashSet<>())
+                .references(new HashSet<>())
+                .matchedRule(-1)
+                .redactionReason("")
+                .legalBasis("")
+                .build();
+    }
+
+
+    // initial values
+    final Boundary boundary;
+    final String type;
+    final EntityType entityType;
+
+    // empty defaults
+    boolean redaction;
+    boolean falsePositive;
+    boolean removed;
+    boolean ignored;
+    boolean resized;
+    boolean skipRemoveEntitiesContainedInLarger;
+    boolean dictionaryEntry;
+    boolean dossierDictionaryEntry;
+    Set<Engine> engines;
+    Set<EntityNode> references;
+    int matchedRule;
+    String redactionReason;
+    String legalBasis;
+
+    // inferred on graph insertion
+    String value;
+    CharSequence textBefore;
+    CharSequence textAfter;
+    @Builder.Default
+    Set<PageNode> pages = new HashSet<>();
+    List<EntityPosition> entityPositions;
+    @Builder.Default
+    List<SemanticNode> intersectingNodes = new LinkedList<>();
+    SemanticNode deepestFullyContainingNode;
+
+
+    public void addIntersectingNode(SemanticNode containingNode) {
+
+        intersectingNodes.add(containingNode);
+    }
+
+
+    public void removeFromGraph() {
+
+        getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
+        deepestFullyContainingNode = null;
+        getPages().forEach(page -> page.getEntities().remove(this));
+        setRemoved(true);
+    }
+
+
+    public List<EntityPosition> getEntityPositions() {
+
+        if (entityPositions == null || entityPositions.isEmpty()) {
+            entityPositions = deepestFullyContainingNode.buildTextBlock().getEntityPositions(boundary);
+        }
+
+        return entityPositions;
+    }
+
+
+    public boolean containedBy(EntityNode entityNode) {
+
+        return this.boundary.containedBy(entityNode.getBoundary());
+    }
+
+
+    public boolean contains(EntityNode entityNode) {
+
+        return this.boundary.contains(entityNode.getBoundary());
+    }
+
+
+    public void addEngine(Engine engine) {
+
+        engines.add(engine);
+    }
+
+    public void addEngines(Set<Engine> engines) {
+
+        this.engines.addAll(engines);
+    }
+
+
+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("Entity[\"");
+        sb.append(value);
+        sb.append("\", ");
+        sb.append(boundary);
+        sb.append(", pages[");
+        pages.forEach(page -> {
+            sb.append(page.getNumber());
+            sb.append(", ");
+        });
+        sb.delete(sb.length() - 2, sb.length());
+        sb.append("], type = \"");
+        sb.append(type);
+        sb.append("\", EntityType.");
+        sb.append(entityType);
+        sb.append("]");
+        return sb.toString();
+    }
+
+
+    @Override
+    public int hashCode() {
+
+        return Hashing.murmur3_128().hashString(toString(), StandardCharsets.UTF_8).hashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+
+        return o.hashCode() == hashCode();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java

@@ -0,0 +1,36 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
+
+import java.awt.geom.Rectangle2D;
+import java.nio.charset.StandardCharsets;
+
+import com.google.common.hash.Hashing;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class EntityPosition {
+
+    PageNode pageNode;
+    Rectangle2D position;
+
+
+    public String getId() {
+
+        return String.valueOf(hashCode());
+    }
+
+
+    @Override
+    public int hashCode() {
+
+        String string = String.valueOf(pageNode.getNumber()) + position.getX() + position.getY() + position.getWidth() + position.getHeight();
+        return Hashing.murmur3_128().hashString(string, StandardCharsets.UTF_8).hashCode();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+package com.iqser.red.service.redaction.v1.server.document.graph.entity;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -10,6 +10,9 @@ import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
@@ -77,6 +80,7 @@ public class ImageNode extends PageElement implements SemanticNode {
         return tocId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
     }
 
+
     @Override
     public Map<PageNode, Rectangle2D> getBBox() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java

@@ -34,7 +34,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
@@ -91,7 +91,7 @@ public class DocumentGraphFactory {
     private void addSection(SemanticNode parentNode, List<AbstractTextContainer> pageBlocks, List<PdfImage> images, Context context) {
 
         Map<Integer, List<AbstractTextContainer>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractTextContainer::getPage));
-        SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).pages(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
+        SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
 
         context.sections().add(sectionNode);
         blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber));
@@ -129,6 +129,7 @@ public class DocumentGraphFactory {
     private List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) {
 
         return pageBlocks.stream()
+                .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
                 .filter(abstractTextContainer -> abstractTextContainer instanceof TextBlock)
                 .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
                 .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
@@ -137,12 +138,9 @@ public class DocumentGraphFactory {
     }
 
 
-
-
     private void addSectionNodeToPageNode(Context context, SectionNode sectionNode, Integer pageNumber) {
 
         PageNode page = getPage(pageNumber, context);
-        sectionNode.getPages().add(page);
         page.getMainBody().add(sectionNode);
     }
 
@@ -150,12 +148,7 @@ public class DocumentGraphFactory {
     private void addTable(SemanticNode parentNode, Table table, Context context) {
 
         PageNode page = getPage(table.getPage(), context);
-        TableNode tableNode = TableNode.builder()
+        TableNode tableNode = TableNode.builder().tableOfContents(context.tableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build();
-                .tableOfContents(context.tableOfContents())
-                .pages(Collections.singleton(page))
-                .numberOfCols(table.getColCount())
-                .numberOfRows(table.getRowCount())
-                .build();
 
         if (!page.getMainBody().contains(parentNode)) {
             parentNode.getPages().add(page);
@@ -166,32 +159,32 @@ public class DocumentGraphFactory {
         List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE, tableNode);
         tableNode.setTocId(tocId);
 
-        addTableCells(table.getRows(), tableNode, context);
+        addTableCells(table.getRows(), tableNode, context, table.getPage());
     }
 
 
-    private void addTableCells(List<List<Cell>> rows, SemanticNode parentNode, Context context) {
+    private void addTableCells(List<List<Cell>> rows, SemanticNode parentNode, Context context, int pageNumber) {
 
         for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
             for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, parentNode, context);
+                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, parentNode, pageNumber, context);
             }
         }
     }
 
 
-    private void addTableCell(Cell cell, int rowIndex, int colIndex, SemanticNode parentNode, Context context) {
+    private void addTableCell(Cell cell, int rowIndex, int colIndex, SemanticNode parentNode, int pageNumber, Context context) {
+
+        PageNode page = getPage(pageNumber, context);
+        cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(pageNumber));
 
-        PageNode page = parentNode.getFirstPage();
         TableCellNode tableCellNode = TableCellNode.builder()
                 .tableOfContents(context.tableOfContents())
                 .row(rowIndex)
                 .col(colIndex)
                 .header(cell.isHeaderCell())
-                .pages(new HashSet<>())
                 .bBox(cell.getBounds2D())
                 .build();
-        tableCellNode.getPages().add(page);
         page.getMainBody().add(tableCellNode);
 
         com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock;
@@ -230,22 +223,15 @@ public class DocumentGraphFactory {
 
     private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context, List<TextBlock> textBlocksToMerge) {
 
-        PageNode page;
+        PageNode page = getPage(originalTextBlock.getPage(), context);
-        if (originalTextBlock.getPage() == 0) {
+
-            page = parentNode.getFirstPage();
-        } else {
-            page = getPage(originalTextBlock.getPage(), context);
-        }
         SemanticNode node;
         if (StringUtils.startsWith(originalTextBlock.getClassification(), "H")) {
-            node = HeadlineNode.builder().pages(Collections.singleton(page)).tableOfContents(context.tableOfContents()).build();
+            node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build();
         } else {
-            node = ParagraphNode.builder().pages(Collections.singleton(page)).tableOfContents(context.tableOfContents()).build();
+            node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build();
         }
 
-        if (!page.getMainBody().contains(parentNode)) {
-            parentNode.getPages().add(page);
-        }
         page.getMainBody().add(node);
 
         List<TextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
@@ -317,7 +303,7 @@ public class DocumentGraphFactory {
     private void addFooter(List<TextBlock> textBlocks, Context context) {
 
         PageNode page = getPage(textBlocks.get(0).getPage(), context);
-        FooterNode footer = FooterNode.builder().page(page).tableOfContents(context.tableOfContents()).build();
+        FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
         AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), footer, context, page);
         List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer);
         footer.setTocId(tocId);
@@ -329,7 +315,7 @@ public class DocumentGraphFactory {
     public void addHeader(List<TextBlock> textBlocks, Context context) {
 
         PageNode page = getPage(textBlocks.get(0).getPage(), context);
-        HeaderNode header = HeaderNode.builder().page(page).tableOfContents(context.tableOfContents()).build();
+        HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
         AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, context, 0, page);
         List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header);
         header.setTocId(tocId);
@@ -341,7 +327,7 @@ public class DocumentGraphFactory {
     private void addEmptyFooter(int pageIndex, Context context) {
 
         PageNode page = getPage(pageIndex, context);
-        FooterNode footer = FooterNode.builder().page(page).tableOfContents(context.tableOfContents()).build();
+        FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
         AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page);
         List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer);
         footer.setTocId(tocId);
@@ -353,7 +339,7 @@ public class DocumentGraphFactory {
     private void addEmptyHeader(int pageIndex, Context context) {
 
         PageNode page = getPage(pageIndex, context);
-        HeaderNode header = HeaderNode.builder().page(page).tableOfContents(context.tableOfContents()).build();
+        HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
         AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, context, 0, page);
         List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header);
         header.setTocId(tocId);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java

@@ -1,7 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.document.graph.factory;
 
+import java.awt.geom.Area;
 import java.awt.geom.Rectangle2D;
 import java.util.List;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
@@ -17,16 +24,19 @@ public class RectangleTransformations {
 
     public static Rectangle2D bBoxUnionAbstractTextContainer(List<AbstractTextContainer> abstractTextContainers) {
 
-        return abstractTextContainers.stream().map(RectangleTransformations::toRectangle2D).reduce((a, b) -> a.createUnion(b).getBounds2D()).orElseThrow();
+        return abstractTextContainers.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion());
     }
 
 
     public static Rectangle2D bBoxUnionAtomicTextBlock(List<AtomicTextBlock> atomicTextBlocks) {
 
-        return atomicTextBlocks.stream()
+        return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DUnion());
-                .flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream())
+    }
-                .reduce((a, b) -> a.createUnion(b).getBounds2D())
+
-                .orElse(new Rectangle2D.Double());
+
+    public static Rectangle2D rectangleUnion(List<Rectangle2D> rectangle2DList) {
+
+        return rectangle2DList.stream().collect(new Rectangle2DUnion());
     }
 
 
@@ -41,4 +51,46 @@ public class RectangleTransformations {
         return new Rectangle2D.Double(redRectangle2D.getX(), redRectangle2D.getY(), redRectangle2D.getWidth(), redRectangle2D.getHeight());
     }
 
+
+    private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
+
+        @Override
+        public Supplier<Area> supplier() {
+
+            return Area::new;
+        }
+
+
+        @Override
+        public BiConsumer<Area, Rectangle2D> accumulator() {
+
+            return (a, b) -> a.add(new Area(b));
+        }
+
+
+        @Override
+        public BinaryOperator<Area> combiner() {
+
+            return (area1, area2) -> {
+                area1.add(area2);
+                return area1;
+            };
+        }
+
+
+        @Override
+        public Function<Area, Rectangle2D> finisher() {
+
+            return Area::getBounds2D;
+        }
+
+
+        @Override
+        public Set<Characteristics> characteristics() {
+
+            return Set.of(Characteristics.CONCURRENT, Characteristics.UNORDERED);
+        }
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java

@@ -52,10 +52,9 @@ public class SearchTextWithTextPositionFactory {
                         stringIdxToPositionIdx = stringIdxToPositionIdx.subList(0, lastHyphenIdx);
                         stringIdx = lastHyphenIdx;
                         lastHyphenIdx = -3;
-
-                    } else {
-                        lineBreaksStringIdx.add(stringIdx);
                     }
+                    lineBreaksStringIdx.add(stringIdx);
+
                 }
                 if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java

@@ -1,127 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.nio.charset.StandardCharsets;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import com.google.common.hash.Hashing;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class EntityNode {
-
-    public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) {
-
-        return EntityNode.builder().type(type).entityType(entityType).boundary(boundary).build();
-    }
-
-
-    // initial values
-    Boundary boundary;
-    String type;
-    EntityType entityType;
-
-    @Builder.Default
-    boolean redaction = false;
-    @Builder.Default
-    boolean falsePositive = false;
-    @Builder.Default
-    boolean removed = false;
-    @Builder.Default
-    boolean ignored = false;
-    @Builder.Default
-    boolean resized = false;
-    @Builder.Default
-    boolean skipRemoveEntitiesContainedInLarger = false;
-    @Builder.Default
-    boolean isDictionaryEntry = false;
-    @Builder.Default
-    Set<Engine> engines = new HashSet<>();
-    @Builder.Default
-    Set<Entity> references = new HashSet<>();
-    @Builder.Default
-    int matchedRule = -1;
-    @Builder.Default
-    String redactionReason = "";
-    @Builder.Default
-    String legalBasis = "";
-
-    // inferred on graph insertion
-    String value;
-    CharSequence textBefore;
-    CharSequence textAfter;
-    @Builder.Default
-    Set<PageNode> pages = new HashSet<>();
-    List<Rectangle2D> positions;
-    @Builder.Default
-    List<SemanticNode> intersectingNodes = new LinkedList<>();
-    SemanticNode deepestFullyContainingNode;
-
-
-    public void addIntersectingNode(SemanticNode containingNode) {
-
-        intersectingNodes.add(containingNode);
-    }
-
-
-    public void removeFromGraph() {
-
-        getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        deepestFullyContainingNode = null;
-        getPages().forEach(page -> page.getEntities().remove(this));
-        setRemoved(true);
-    }
-
-
-    @Override
-    public String toString() {
-
-        StringBuilder sb = new StringBuilder();
-        sb.append("Entity [");
-        sb.append(value);
-        sb.append(", ");
-        sb.append(boundary);
-        sb.append(", Pages[");
-        pages.forEach(page -> {
-            sb.append(page.getNumber());
-            sb.append(", ");
-        });
-        sb.delete(sb.length() - 2, sb.length());
-        sb.append("], containingNode ");
-        sb.append(deepestFullyContainingNode);
-        return sb.toString();
-    }
-
-
-    @Override
-    public int hashCode() {
-
-        StringBuilder sb = new StringBuilder();
-        sb.append("Entity [");
-        sb.append(value);
-        sb.append(", ");
-        sb.append(boundary);
-        sb.append(", Pages[");
-        pages.forEach(page -> {
-            sb.append(page.getNumber());
-            sb.append(", ");
-        });
-        return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java

@@ -1,12 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
@@ -30,9 +30,6 @@ public class FooterNode extends PageElement implements SemanticNode {
     @Builder.Default
     boolean terminal = true;
 
-    @EqualsAndHashCode.Exclude
-    PageNode page;
-
     @EqualsAndHashCode.Exclude
     TableOfContents tableOfContents;
 
@@ -48,13 +45,6 @@ public class FooterNode extends PageElement implements SemanticNode {
     }
 
 
-    @Override
-    public Set<PageNode> getPages() {
-
-        return Collections.singleton(page);
-    }
-
-
     @Override
     public String toString() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java

@@ -1,12 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
@@ -30,9 +30,6 @@ public class HeaderNode extends PageElement implements SemanticNode {
     @Builder.Default
     boolean terminal = true;
 
-    @EqualsAndHashCode.Exclude
-    PageNode page;
-
     @EqualsAndHashCode.Exclude
     TableOfContents tableOfContents;
 
@@ -47,12 +44,6 @@ public class HeaderNode extends PageElement implements SemanticNode {
         return terminalTextBlock;
     }
 
-    @Override
-    public Set<PageNode> getPages() {
-
-        return Collections.singleton(page);
-    }
-
     @Override
     public String toString() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java

@@ -6,6 +6,7 @@ import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
@@ -32,9 +33,6 @@ public class HeadlineNode extends PageElement implements SemanticNode {
     @EqualsAndHashCode.Exclude
     TableOfContents tableOfContents;
 
-    @EqualsAndHashCode.Exclude
-    Set<PageNode> pages;
-
     @Builder.Default
     @EqualsAndHashCode.Exclude
     Set<EntityNode> entities = new HashSet<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java

@@ -4,11 +4,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java

@@ -6,6 +6,7 @@ import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
@@ -30,9 +31,6 @@ public class ParagraphNode extends PageElement implements SemanticNode {
     @EqualsAndHashCode.Exclude
     TableOfContents tableOfContents;
 
-    @EqualsAndHashCode.Exclude
-    Set<PageNode> pages;
-
     @Builder.Default
     @EqualsAndHashCode.Exclude
     Set<EntityNode> entities = new HashSet<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java

@@ -6,6 +6,7 @@ import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
@@ -30,8 +31,6 @@ public class SectionNode extends PageElement implements SemanticNode {
     TextBlock textBlock;
     @EqualsAndHashCode.Exclude
     TableOfContents tableOfContents;
-    @EqualsAndHashCode.Exclude
-    Set<PageNode> pages;
 
     @Builder.Default
     @EqualsAndHashCode.Exclude

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java

@@ -10,7 +10,9 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Stream;
 
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@@ -28,14 +30,17 @@ public interface SemanticNode {
 
     /**
      * Any Node maintains its own Set of Entities.
-     * This Set contains all Entities, whose boundary intersects the boundary of the entity.
+     * This Set contains all Entities whose boundary intersects the boundary of this node.
      *
      * @return Set of all Entities associated with this Node
      */
     Set<EntityNode> getEntities();
 
 
-    Set<PageNode> getPages();
+    default Set<PageNode> getPages() {
+
+        return buildTextBlock().getPages();
+    }
 
 
     default PageNode getFirstPage() {
@@ -117,6 +122,12 @@ public interface SemanticNode {
     }
 
 
+    default boolean containsString(String string) {
+
+        return buildTextBlock().getSearchText().contains(string);
+    }
+
+
     default void addThisToEntityIfIntersects(EntityNode entity) {
 
         TextBlock textBlock = buildTextBlock();
@@ -148,15 +159,33 @@ public interface SemanticNode {
 
         Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
         if (isTerminal()) {
-            Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage));
+            return getBBoxFromTerminalTextBlock(bBoxPerPage);
-            atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
-            return bBoxPerPage;
         }
 
+        return getBBoxFromChildren(bBoxPerPage);
+    }
+
+
+    default Boundary getBoundary() {
+
+        return buildTextBlock().getBoundary();
+    }
+
+
+    private Map<PageNode, Rectangle2D> getBBoxFromChildren(Map<PageNode, Rectangle2D> bBoxPerPage) {
+
         return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
             map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
             return map2;
         }).orElse(bBoxPerPage);
     }
 
+
+    private Map<PageNode, Rectangle2D> getBBoxFromTerminalTextBlock(Map<PageNode, Rectangle2D> bBoxPerPage) {
+
+        Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage));
+        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
+        return bBoxPerPage;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java

@@ -10,6 +10,7 @@ import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
@@ -42,9 +43,6 @@ public class TableCellNode extends PageElement implements SemanticNode {
     @EqualsAndHashCode.Exclude
     TableOfContents tableOfContents;
 
-    @EqualsAndHashCode.Exclude
-    Set<PageNode> pages;
-
     @Builder.Default
     @EqualsAndHashCode.Exclude
     Set<EntityNode> entities = new HashSet<>();
@@ -54,10 +52,11 @@ public class TableCellNode extends PageElement implements SemanticNode {
     public Map<PageNode, Rectangle2D> getBBox() {
 
         Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
-        pages.forEach(page -> bBoxPerPage.put(page, bBox));
+        getPages().forEach(page -> bBoxPerPage.put(page, bBox));
         return bBoxPerPage;
     }
 
+
     @Override
     public TextBlock buildTextBlock() {
 
@@ -67,13 +66,22 @@ public class TableCellNode extends PageElement implements SemanticNode {
         return textBlock;
     }
 
+
     @Override
     public String toString() {
 
         return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().getSearchText();
     }
 
-    public Stream<TableCellNode> getHeaders() {
+
+    public boolean hasHeader(String headerString) {
+
+        return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().contains(headerString));
+    }
+
+
+    private Stream<TableCellNode> getHeaders() {
+
         TableNode tableNode = (TableNode) getParent();
         return tableNode.streamHeadersForCell(row, col);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java

@@ -7,6 +7,7 @@ import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.PageElement;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 
@@ -30,8 +31,6 @@ public class TableNode extends PageElement implements SemanticNode {
     Integer numberOfCols;
 
     TextBlock textBlock;
-    @EqualsAndHashCode.Exclude
-    Set<PageNode> pages;
 
     @Builder.Default
     @EqualsAndHashCode.Exclude

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java

@@ -3,11 +3,14 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
 import java.util.List;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
+import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -42,7 +45,21 @@ public class AtomicTextBlock implements TextBlock {
     @Override
     public int numberOfLines() {
 
-        return lineBreaks.size();
+        return lineBreaks.size() + 1;
+    }
+
+
+    public CharSequence getLine(int lineNumber) {
+
+        if (lineNumber >= numberOfLines() || lineNumber < 0) {
+            throw new IndexOutOfBoundsException(format("line %d out of range for AtomicTextBlock with %d lines", lineNumber, numberOfLines()));
+        }
+        if (lineNumber == 0) {
+            return subSequence(boundary.start(), lineBreaks.get(0) + boundary.start());
+        } else if (lineNumber == numberOfLines() - 1) {
+            return subSequence(lineBreaks.get(lineBreaks.size() - 1) + boundary.start(), boundary.end());
+        }
+        return subSequence(lineBreaks.get(lineNumber - 1) + boundary.start(), lineBreaks.get(lineNumber) + boundary.start());
     }
 
 
@@ -81,17 +98,34 @@ public class AtomicTextBlock implements TextBlock {
 
 
     @Override
-    public List<Rectangle2D> getPositions(Boundary boundary) {
+    public List<Rectangle2D> getPositions(Boundary stringBoundary) {
 
-        if (!containsBoundary(boundary)) {
+        if (!containsBoundary(stringBoundary)) {
-            throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", boundary, this.boundary));
+            throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", stringBoundary, this.boundary));
         }
 
-        if (boundary.end() == this.boundary.end()) {
+        if (stringBoundary.end() == this.boundary.end()) {
-            return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), positions.size());
+            return positions.subList(stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start()), positions.size());
         }
 
-        return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), stringIdxToPositionIdx.get(boundary.end() - this.boundary.start()));
+        return positions.subList(stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start()),
+                stringIdxToPositionIdx.get(stringBoundary.end() - this.boundary.start()));
+    }
+
+
+    public List<EntityPosition> getEntityPositions(Boundary stringBoundary) {
+
+        List<Rectangle2D> positionsPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList())
+                .stream()
+                .map(this::getPositions)
+                .map(RectangleTransformations::rectangleUnion)
+                .toList();
+
+        List<EntityPosition> entityPositions = new LinkedList<>();
+        for (Rectangle2D position : positionsPerLine) {
+            entityPositions.add(EntityPosition.builder().position(position).pageNode(page).build());
+        }
+        return entityPositions;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java

@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.function.Supplier;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
 
 import lombok.AccessLevel;
 import lombok.Data;
@@ -57,7 +58,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
     }
 
 
-    private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringIdxRange(Boundary boundary) {
+    private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(Boundary boundary) {
 
         return atomicTextBlocks.stream().filter(tb -> tb.getBoundary().intersects(boundary)).toList();
     }
@@ -95,6 +96,12 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
         return getAtomicTextBlockByStringIndex(fromIndex).getPreviousLinebreak(fromIndex);
     }
 
+    @Override
+    public List<Integer> getLineBreaks() {
+
+        return getAtomicTextBlocks().stream().flatMap(atomicTextBlock -> atomicTextBlock.getLineBreaks().stream()).toList();
+    }
+
 
     @Override
     public Rectangle2D getPosition(int stringIdx) {
@@ -104,23 +111,46 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTe
 
 
     @Override
-    public List<Rectangle2D> getPositions(Boundary boundary) {
+    public List<Rectangle2D> getPositions(Boundary stringBoundary) {
 
-        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringIdxRange(boundary);
+        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
 
         if (textBlocks.size() == 1) {
-            return textBlocks.get(0).getPositions(boundary);
+            return textBlocks.get(0).getPositions(stringBoundary);
         }
 
         AtomicTextBlock firstTextBlock = textBlocks.get(0);
-        List<Rectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(new Boundary(boundary.start(), firstTextBlock.getBoundary().end())));
+        List<Rectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
 
         for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
             positions.addAll(textBlock.getPositions());
         }
 
         var lastTextBlock = textBlocks.get(textBlocks.size() - 1);
-        positions.addAll(lastTextBlock.getPositions(new Boundary(lastTextBlock.getBoundary().start(), boundary.end())));
+        positions.addAll(lastTextBlock.getPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
+
+        return positions;
+    }
+
+
+    @Override
+    public List<EntityPosition> getEntityPositions(Boundary stringBoundary) {
+
+        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
+
+        if (textBlocks.size() == 1) {
+            return textBlocks.get(0).getEntityPositions(stringBoundary);
+        }
+
+        AtomicTextBlock firstTextBlock = textBlocks.get(0);
+        List<EntityPosition> positions = new LinkedList<>(firstTextBlock.getEntityPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
+
+        for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
+            positions.addAll(textBlock.getEntityPositions(textBlock.getBoundary()));
+        }
+
+        AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
+        positions.addAll(lastTextBlock.getEntityPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
 
         return positions;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java

@@ -4,8 +4,12 @@ import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 
 public interface TextBlock extends CharSequence {
 
@@ -23,21 +27,30 @@ public interface TextBlock extends CharSequence {
 
     int getPreviousLinebreak(int fromIndex);
 
+    List<Integer> getLineBreaks();
+
 
     Rectangle2D getPosition(int stringIdx);
 
 
-    List<Rectangle2D> getPositions(Boundary range);
+    List<Rectangle2D> getPositions(Boundary stringBoundary);
+
+
+    List<EntityPosition> getEntityPositions(Boundary stringBoundary);
 
 
     int numberOfLines();
 
-
     default int indexOf(String searchTerm) {
 
         return indexOf(searchTerm, getBoundary().start());
     }
 
+    default Set<PageNode> getPages() {
+
+        return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet());
+    }
+
 
     default int indexOf(String searchTerm, int startOffset) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java

@@ -2,19 +2,22 @@ package com.iqser.red.service.redaction.v1.server.document.services;
 
 import static java.lang.String.format;
 
+import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -24,10 +27,29 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class EntityCreationService {
 
-    private final EntityEnrichmentService entityEnrichmentService;
+    private final EntityTextEnrichmentService entityEnrichmentService;
 
 
-    public EntityNode createAndAddEntity(Boundary boundary, String type, EntityType entityType, DocumentGraph documentGraph) {
+    public Set<EntityNode> createEntitiesByLineAfterString(String string, SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) {
+
+        TextBlock textBlock = node.buildTextBlock();
+        SearchImplementation searchImplementation = new SearchImplementation(List.of(string), true);
+        List<Boundary> boundaries = searchImplementation.getBoundaries(textBlock, node.getBoundary());
+        return boundaries.stream()
+                .map(boundary -> toLineAfterBoundary(textBlock, boundary))
+                .map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph))
+                .collect(Collectors.toUnmodifiableSet());
+    }
+
+
+    public Set<EntityNode> createEntitiesByRegex(String regexPattern, String type, EntityType entityType, DocumentGraph documentGraph) {
+
+        List<Boundary> boundaries = RegexMatcher.findBoundaries(regexPattern, documentGraph.buildTextBlock());
+        return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph)).collect(Collectors.toUnmodifiableSet());
+    }
+
+
+    public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, DocumentGraph documentGraph) {
 
         EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType);
         addEntityToGraph(entity, documentGraph);
@@ -35,7 +57,21 @@ public class EntityCreationService {
     }
 
 
-    public EntityNode createAndAddEntity(String entityName, int startOffset, String type, EntityType entityType, DocumentGraph documentGraph) {
+    public EntityNode createEntityByEntity(EntityNode entity, String type, EntityType entityType, DocumentGraph documentGraph) {
+
+        return createEntityByBoundary(entity.getBoundary(), type, entityType, documentGraph);
+    }
+
+
+    public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) {
+
+        Boundary boundary = node.buildTextBlock().getBoundary();
+        Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
+        return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, documentGraph);
+    }
+
+
+    public Set<EntityNode> createEntitiesByString(String entityName, int startOffset, String type, EntityType entityType, DocumentGraph documentGraph) {
 
         int start = documentGraph.buildTextBlock().indexOf(entityName, startOffset);
         if (start == -1) {
@@ -43,7 +79,7 @@ public class EntityCreationService {
         }
         EntityNode entity = EntityNode.initialEntityNode(new Boundary(start, start + entityName.length()), type, entityType);
         addEntityToGraph(entity, documentGraph);
-        return entity;
+        return Set.of(entity);
     }
 
 
@@ -87,4 +123,9 @@ public class EntityCreationService {
         entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
     }
 
+    private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) {
+
+        return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end()));
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityTextEnrichmentService.java

@@ -6,7 +6,7 @@ import java.util.Objects;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
@@ -14,16 +14,16 @@ import lombok.RequiredArgsConstructor;
 
 @Service
 @RequiredArgsConstructor
-public class EntityEnrichmentService {
+public class EntityTextEnrichmentService {
 
     private final RedactionServiceSettings redactionServiceSettings;
 
+
     public EntityNode enrichEntity(EntityNode entity, TextBlock textBlock) {
 
-        entity.setPositions(textBlock.getPositions(entity.getBoundary()));
+        entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
         entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock));
         entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock));
-        entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
         return entity;
     }
 
@@ -43,8 +43,6 @@ public class EntityEnrichmentService {
     }
 
 
-
-
     private CharSequence findTextBefore(int index, TextBlock textBlock) {
 
         int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().start());
@@ -65,11 +63,11 @@ public class EntityEnrichmentService {
         return Arrays.stream(textAfter.split(" ")).filter(word -> !Objects.equals("", word)).toList();
     }
 
+
     private String concatWordsBefore(List<String> words, boolean endWithSpace) {
 
         StringBuilder sb = new StringBuilder();
 
-
         for (String word : words) {
             sb.append(word).append(" ");
         }
@@ -78,11 +76,12 @@ public class EntityEnrichmentService {
         return endWithSpace ? result + " " : result;
     }
 
+
     private String concatWordsAfter(List<String> words, boolean startWithSpace) {
 
         StringBuilder sb = new StringBuilder();
 
-        for (String word: words) {
+        for (String word : words) {
             sb.append(word).append(" ");
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ImageSortService.java

@@ -1,8 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.document.services;
 
+import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.bBoxUnionAbstractTextContainer;
+import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle2D;
 import static java.util.stream.Collectors.groupingBy;
 
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -63,9 +64,6 @@ public class ImageSortService {
                 return;
             }
         }
-        if (sectionContainingTextContainer.isPresent()) {
-
-        }
     }
 
 
@@ -89,21 +87,24 @@ public class ImageSortService {
 
     private static Optional<Cell> getContainingCell(Table table, PdfImage image) {
 
-        return table.getRows().stream().flatMap(List::stream).filter(cell -> cell.contains(toRectangle2D(image))).findFirst();
+        return table.getRows().stream().flatMap(List::stream).filter(cell -> cell.contains(toRectangle2D(image.getPosition()))).findFirst();
     }
 
 
     private List<Section> getContainedSections(PdfImage image, List<Section> sectionsOnPage) {
 
         return sectionsOnPage.stream()
-                .filter(section -> toRectangle2D(image).contains(bBoxUnion(section.getPageBlocks().stream().filter(block -> block.getPage() == image.getPage()).toList())))
+                .filter(section -> toRectangle2D(image.getPosition()).contains(bBoxUnionAbstractTextContainer(section.getPageBlocks()
+                        .stream()
+                        .filter(block -> block.getPage() == image.getPage())
+                        .toList())))
                 .toList();
     }
 
 
     private List<AbstractTextContainer> getContainedTextContainers(PdfImage image, List<AbstractTextContainer> textContainersOnPage) {
 
-        return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(image).contains(toRectangle2D(textContainer))).toList();
+        return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(image.getPosition()).contains(toRectangle2D(textContainer))).toList();
     }
 
 
@@ -111,33 +112,14 @@ public class ImageSortService {
 
         return sectionsOnPage.stream()//
                 .filter(section -> //
-                        bBoxUnion(section.getPageBlocks().stream().filter(block -> block.getPage() == image.getPage()).toList())//
+                        bBoxUnionAbstractTextContainer(section.getPageBlocks().stream().filter(block -> block.getPage() == image.getPage()).toList())//
-                                .contains(toRectangle2D(image))).findFirst();
+                                .contains(toRectangle2D(image.getPosition()))).findFirst();
     }
 
 
     private Optional<AbstractTextContainer> getContainingTextContainer(PdfImage image, List<AbstractTextContainer> textContainersOnPage) {
 
-        return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(textContainer).contains(toRectangle2D(image))).findFirst();
+        return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(textContainer).contains(toRectangle2D(image.getPosition()))).findFirst();
-    }
-
-
-    private static Rectangle2D toRectangle2D(AbstractTextContainer container) {
-
-        return new Rectangle2D.Double(container.getMinX(), container.getMinY(), container.getWidth(), container.getHeight());
-    }
-
-
-    private static Rectangle2D toRectangle2D(PdfImage image) {
-
-        var redRectangle2D = image.getPosition();
-        return new Rectangle2D.Double(redRectangle2D.getX(), redRectangle2D.getY(), redRectangle2D.getWidth(), redRectangle2D.getHeight());
-    }
-
-
-    private static Rectangle2D bBoxUnion(List<AbstractTextContainer> blocks) {
-
-        return blocks.stream().map(ImageSortService::toRectangle2D).reduce((a, b) -> a.createUnion(b).getBounds2D()).orElse(new Rectangle2D.Double());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.document.services;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
@@ -11,21 +12,21 @@ public class RegexMatcher {
 
     public static boolean anyMatch(CharSequence searchText, String regexPattern) {
 
-        var pattern = Patterns.getCompiledPattern(regexPattern, false);
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
         return pattern.matcher(searchText).find();
     }
 
 
     public static Boundary findFirstBoundary(String regexPattern, CharSequence searchText) {
 
-        var pattern = Patterns.getCompiledPattern(regexPattern, false);
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
         Matcher matcher = pattern.matcher(searchText);
         return new Boundary(matcher.start(), matcher.end());
     }
 
     public static List<Boundary> findBoundaries(String regexPattern, CharSequence searchText) {
 
-        var pattern = Patterns.getCompiledPattern(regexPattern, false);
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
         Matcher matcher = pattern.matcher(searchText);
         List<Boundary> boundaries = new LinkedList<>();
         while (matcher.find()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,15 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.*;
-
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
 @Data
 @Builder
 @NoArgsConstructor
@@ -66,7 +70,6 @@ public class Entity implements ReasonHolder {
                   String headline,
                   int matchedRule,
                   int sectionNumber,
-                  int paragraphNumber,
                   String legalBasis,
                   boolean isDictionaryEntry,
                   String textBefore,
@@ -86,7 +89,6 @@ public class Entity implements ReasonHolder {
         this.headline = headline;
         this.matchedRule = matchedRule;
         this.sectionNumber = sectionNumber;
-        this.paragraphNumber = paragraphNumber;
         this.legalBasis = legalBasis;
         this.isDictionaryEntry = isDictionaryEntry;
         this.textBefore = textBefore;
@@ -106,7 +108,6 @@ public class Entity implements ReasonHolder {
                   Integer end,
                   String headline,
                   int sectionNumber,
-                  int paragraphNumber,
                   boolean isDictionaryEntry,
                   boolean isDossierDictionaryEntry,
                   Engine engine,
@@ -118,7 +119,6 @@ public class Entity implements ReasonHolder {
         this.end = end;
         this.headline = headline;
         this.sectionNumber = sectionNumber;
-        this.paragraphNumber = paragraphNumber;
         this.isDictionaryEntry = isDictionaryEntry;
         this.isDossierDictionaryEntry = isDossierDictionaryEntry;
         this.engines.add(engine);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -1363,7 +1363,6 @@ public class Section {
                     value.getRowSpanStart() + word.length(),
                     headline,
                     sectionNumber,
-                    -1,
                     false,
                     false,
                     Engine.RULE,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java

@@ -0,0 +1,16 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class SectionSearchableTextPair {
+
+    private Section section;
+    private SearchableText searchableText;
+    private List<Integer> cellStarts;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -104,6 +104,18 @@ public class DroolsExecutionService {
 
     }
 
+    @Timed("redactmanager_executeRules")
+    public Section executeRules(KieContainer kieContainer, Section section) {
+
+        KieSession kieSession = kieContainer.newKieSession();
+        kieSession.insert(section);
+        kieSession.fireAllRules();
+        kieSession.dispose();
+
+        return section;
+
+    }
+
     public List<Entity> getEntities(KieSession ks) {
         List<Entity> entities = new LinkedList<>();
         QueryResults entitiesResult = ks.getQueryResults("getEntities");

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -12,6 +13,8 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@@ -32,6 +35,22 @@ public class RedactionLogCreatorService {
     private final DictionaryService dictionaryService;
 
 
+    public List<RedactionLogEntry> createRedactionLog(Map<PageNode, Set<EntityNode>> entityNodesPerPageNode, int numberOfPages, String dossierTemplateId) {
+
+        List<RedactionLogEntry> entries = new ArrayList<>();
+        entityNodesPerPageNode.forEach(((pageNode, entityNodes) -> entries.addAll(toRedactionLogEntries(pageNode, entityNodes, dossierTemplateId))));
+
+
+        return entries;
+    }
+
+
+    private Set<RedactionLogEntry> toRedactionLogEntries(PageNode pageNode, Set<EntityNode> entityNodes, String dossierTemplateId) {
+
+        return Collections.emptySet();
+    }
+
+
     @Timed("redactmanager_createRedactionLog")
     public List<RedactionLogEntry> createRedactionLog(PageEntities pageEntities, int numberOfPages, String dossierTemplateId) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java

@@ -26,12 +26,14 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS
 import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -45,8 +47,8 @@ import lombok.extern.slf4j.Slf4j;
 public class EntityRedactionService {
 
     DroolsExecutionService droolsExecutionService;
-    SurroundingWordsService surroundingWordsService;
     EntityFinder entityFinder;
+    RedactionServiceSettings redactionServiceSettings;
 
 
     public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@@ -78,20 +80,20 @@ public class EntityRedactionService {
         }
 
         Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities());
-        EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
+        //EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
 
         return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes());
     }
 
 
-    public FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
+    private FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
-                                           Dictionary dictionary,
+                                            Dictionary dictionary,
-                                           KieContainer kieContainer,
+                                            KieContainer kieContainer,
-                                           AnalyzeRequest analyzeRequest,
+                                            AnalyzeRequest analyzeRequest,
-                                           boolean local,
+                                            boolean local,
-                                           Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                            Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                           Map<Integer, Set<Image>> imagesPerPage,
+                                            Map<Integer, Set<Image>> imagesPerPage,
-                                           NerEntities nerEntities) {
+                                            NerEntities nerEntities) {
 
         List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
                 dictionary,
@@ -123,14 +125,7 @@ public class EntityRedactionService {
                     .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
                     .collect(Collectors.toSet());
 
-            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
+            addSurroundingText(dictionary, sectionSearchableTextPair.getSearchableText(), sectionSearchableTextPair.getCellStarts(), entriesWithoutSurroundingText);
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                        sectionSearchableTextPair.getSearchableText(),
-                        dictionary,
-                        sectionSearchableTextPair.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
-            }
 
             entities.addAll(analysedSection.getEntities());
 
@@ -147,6 +142,24 @@ public class EntityRedactionService {
     }
 
 
+    private void addSurroundingText(Dictionary dictionary, SearchableText searchableText,  List<Integer> cellStarts, Set<Entity> entriesWithoutSurroundingText) {
+
+        if (cellStarts != null && !cellStarts.isEmpty()) {
+            SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                    searchableText, dictionary,
+                    cellStarts,
+                    redactionServiceSettings.getSurroundingWordsOffsetWindow(),
+                    redactionServiceSettings.getNumberOfSurroundingWords());
+
+        } else {
+            SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                    searchableText, dictionary,
+                    redactionServiceSettings.getSurroundingWordsOffsetWindow(),
+                    redactionServiceSettings.getNumberOfSurroundingWords());
+        }
+    }
+
+
     private List<SectionSearchableTextPair> extractSearchableTextPairs(List<SectionText> reanalysisSections,
                                                                        Dictionary dictionary,
                                                                        AnalyzeRequest analyzeRequest,
@@ -165,11 +178,7 @@ public class EntityRedactionService {
                     reanalysisSection.getCellStarts(),
                     analyzeRequest.getManualRedactions());
 
-            if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
+            addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities());
-                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
-            }
 
             if (!local && analyzeRequest.getManualRedactions() != null) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -63,7 +63,7 @@ public final class EntitySearchUtils {
 
         Set<Entity> entities = new HashSet<>();
 
-        searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.getStartIndex(), match.getEndIndex()));
+        searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.startIndex(), match.endIndex()));
 
         return entities;
     }
@@ -190,7 +190,7 @@ public final class EntitySearchUtils {
     public void removeEntitiesContainedInLarger(Set<Entity> entities) {
 
         List<Entity> wordsToRemove = new ArrayList<>();
-        for (Entity word : entities) {
+        for (Entity outer : entities) {
             for (Entity inner : entities) {
 //                // skip cross-type false positives
 //                if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){
@@ -201,17 +201,19 @@ public final class EntitySearchUtils {
 //                    continue;
 //                }
 
-                if ((inner.isSkipRemoveEntitiesContainedInLarger() || word.isSkipRemoveEntitiesContainedInLarger()) && !inner.getType().equals(word.getType())) {
+                if ((inner.isSkipRemoveEntitiesContainedInLarger() || outer.isSkipRemoveEntitiesContainedInLarger()) && !inner.getType().equals(outer.getType())) {
                     continue;
                 }
 
-                if (inner.getWord().length() < word.getWord()
+                if (inner.getWord().length() < outer.getWord()
-                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
+                        .length() && inner.getStart() >= outer.getStart() && inner.getEnd() <= outer.getEnd() && outer != inner && outer.getSectionNumber() == inner.getSectionNumber()) {
-                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
+                    if (outer.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
-                        wordsToRemove.add(word);
+                        wordsToRemove.add(outer);
-                    } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY || inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
+                    } else if (!(
+                            inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && outer.getEntityType() == EntityType.ENTITY ||
+                            inner.getEntityType() == EntityType.ENTITY && outer.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
                         if (inner.isResized()) {
-                            wordsToRemove.add(word);
+                            wordsToRemove.add(outer);
                         } else {
                             wordsToRemove.add(inner);
                         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.awt.geom.Rectangle2D;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 
@@ -27,6 +28,12 @@ public final class IdBuilder {
     }
 
 
+    public String buildId(Rectangle2D rectangle2D, int page) {
+
+        return hashFunction.hashString("x" + rectangle2D.getX() + "y" + rectangle2D.getY() + "h" + rectangle2D.getHeight() + "w" + rectangle2D.getWidth() + "p" + page,
+                StandardCharsets.UTF_8).toString();
+    }
+
     public String buildId(RedRectangle2D rectangle2D, int page) {
 
         return hashFunction.hashString("x" + rectangle2D.getX() + "y" + rectangle2D.getY() + "h" + rectangle2D.getHeight() + "w" + rectangle2D.getWidth() + "p" + page,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java

@@ -78,7 +78,7 @@ public class SearchImplementation {
     }
 
 
-    public List<Boundary> getMatches(CharSequence text) {
+    public List<Boundary> getBoundaries(CharSequence text) {
 
         if (this.values.isEmpty()) {
             return new ArrayList<>();
@@ -91,7 +91,7 @@ public class SearchImplementation {
     }
 
 
-    public List<Boundary> getMatches(CharSequence text, Boundary region) {
+    public List<Boundary> getBoundaries(CharSequence text, Boundary region) {
 
         if (this.values.isEmpty()) {
             return new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java

@@ -43,14 +43,15 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@@ -67,6 +68,7 @@ public class AbstractTestWithDictionaries {
     protected static final String RULES = loadFromClassPath("drools/rules.drl");
     protected static final String RULES_PATH = "drools/rules.drl";
     protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
+    protected static final String MERGE_ENTITY_RULES_PATH = "drools/merge_entity_rules.drl";
     private static final String VERTEBRATE = "vertebrate";
     private static final String ADDRESS = "CBI_address";
     private static final String AUTHOR = "CBI_author";
@@ -118,6 +120,8 @@ public class AbstractTestWithDictionaries {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
+
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
 
@@ -146,25 +150,6 @@ public class AbstractTestWithDictionaries {
     }
 
 
-    private static String loadFromClassPath(String path) {
-
-        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
-        }
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
-            StringBuilder sb = new StringBuilder();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
-        }
-    }
-
-
     private void mockDictionaryCalls(Long version) {
 
         when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
@@ -205,6 +190,25 @@ public class AbstractTestWithDictionaries {
     }
 
 
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
     private String cleanDictionaryEntry(String entry) {
 
         return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
@@ -357,7 +361,9 @@ public class AbstractTestWithDictionaries {
     private void loadNerForTest() {
 
         ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                responseJson.getInputStream());
     }
 
 
@@ -379,6 +385,7 @@ public class AbstractTestWithDictionaries {
                 .collect(Collectors.toList());
     }
 
+
     protected List<File> getPathsRecursively(File path) {
 
         List<File> result = new ArrayList<>();
@@ -396,6 +403,7 @@ public class AbstractTestWithDictionaries {
 
     }
 
+
     protected void loadOnlyDictionaryForSimpleFile() {
 
         dictionary.clear();
@@ -478,9 +486,13 @@ public class AbstractTestWithDictionaries {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), imageInfoStream);
+        storageService.storeObject(TenantContext.getTenantId(),
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+                imageInfoStream);
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
+                cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
 
         return request;
     }
@@ -505,24 +517,24 @@ public class AbstractTestWithDictionaries {
             KieServices kieServices = KieServices.Factory.get();
 
             KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            //kieFileSystem.write(ResourceFactory.newClassPathResource(RULES_PATH, "UTF-8"));
+            kieFileSystem.write(ResourceFactory.newClassPathResource(MERGE_ENTITY_RULES_PATH, "UTF-8"));
             kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8"));
 
             KieRepository kieRepository = kieServices.getRepository();
 
             kieRepository.addKieModule(new KieModule() {
                 public ReleaseId getReleaseId() {
+
                     return kieRepository.getDefaultReleaseId();
                 }
             });
 
-            KieBuilder kieBuilder = kieServices
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem).buildAll();
-                    .newKieBuilder(kieFileSystem)
-                    .buildAll();
 
             return kieServices.newKieContainer(kieRepository.getDefaultReleaseId());
         }
 
+
         @Bean
         @Primary
         public StorageService inmemoryStorage() {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java

@@ -17,10 +17,12 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.core.io.ClassPathResource;
 
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition;
 import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
@@ -29,7 +31,9 @@ import com.iqser.red.service.redaction.v1.server.document.services.EntityCreatio
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.utils.PdfDraw;
 
@@ -49,13 +53,14 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
     @Autowired
     private EntityCreationService entityCreationService;
 
+    @Autowired
+    private RedactionLogCreatorService redactionLogCreatorService;
+
     @Qualifier("kieContainer")
     @Autowired
     private KieContainer kieContainer;
 
 
-
-
     @Test
     @SneakyThrows
     public void testDroolsOnDocumentGraph() {
@@ -80,16 +85,22 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
         }
 
         KieSession kieSession = kieContainer.newKieSession();
+        entityCreationService.createEntitiesByString("Michael N.", 0, "CBI_author", EntityType.FALSE_POSITIVE, document)//
+                .forEach(kieSession::insert);
         kieSession.setGlobal("document", document);
+        kieSession.setGlobal("entityCreationService", entityCreationService);
 
         document.getEntities().forEach(kieSession::insert);
         document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node()));
         document.getPages().forEach(kieSession::insert);
 
+        long start = System.currentTimeMillis();
         kieSession.insert(FileAttribute.builder().label("Vertebrate Study").value("Yes").build());
 
         kieSession.fireAllRules();
+        System.out.printf("Firing all rules took %d ms\n", System.currentTimeMillis() - start);
 
+        //List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog();
         drawAllEntities(filename, fileResource, document);
     }
 
@@ -168,7 +179,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
             var searchTime = System.currentTimeMillis() - searchStart;
             totalSearchTime += searchTime;
 
-            var insertStart =  System.currentTimeMillis();
+            var insertStart = System.currentTimeMillis();
             DocumentGraph finalDocument = document;
             foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument));
             var insertTime = System.currentTimeMillis() - insertStart;
@@ -196,9 +207,10 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
             for (PageNode page : document.getPages()) {
                 List<Rectangle2D> entityPositionsOnPage = page.getEntities()
                         .stream()
+                        .filter(entityNode -> !entityNode.isRemoved())
                         .filter(EntityNode::isRedaction)
-                        .map(EntityNode::getPositions)
+                        .flatMap(entityNode -> entityNode.getEntityPositions().stream())
-                        .flatMap(List::stream)
+                        .map(EntityPosition::getPosition)
                         .toList();
 
                 PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLACK).stroke(true).build();
@@ -208,9 +220,10 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
             for (PageNode page : document.getPages()) {
                 List<Rectangle2D> entityPositionsOnPage = page.getEntities()
                         .stream()
+                        .filter(entityNode -> !entityNode.isRemoved())
                         .filter(e -> !e.isRedaction())
-                        .map(EntityNode::getPositions)
+                        .flatMap(entityNode -> entityNode.getEntityPositions().stream())
-                        .flatMap(List::stream)
+                        .map(EntityPosition::getPosition)
                         .toList();
 
                 PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLUE).stroke(true).build();
@@ -229,10 +242,19 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
                                                       List<EntityNode> foundEntities,
                                                       String type) {
 
-            TextBlock textBlock = documentGraph.getTextBlock();
+        TextBlock textBlock = documentGraph.getTextBlock();
-            searchImplementation.getMatches(textBlock, textBlock.getBoundary())
+        searchImplementation.getBoundaries(textBlock, textBlock.getBoundary())
-                    .stream()
+                .stream()
-                    .map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType))
+                .filter(boundary -> validateSeparators(textBlock, boundary))
-                    .forEach(foundEntities::add);
+                .map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType))
+                .forEach(foundEntities::add);
     }
+
+
+    private static boolean validateSeparators(TextBlock textBlock, Boundary boundary) {
+
+        return (boundary.start() == 0 || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start()))) && (boundary.end() == textBlock.length() || SeparatorUtils.isSeparator(
+                textBlock.charAt(boundary.end())) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1)));
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java

@@ -1,6 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.document.graph;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Collections;
+import java.util.List;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -8,14 +14,18 @@ import org.junit.jupiter.api.Test;
 class BoundaryTest {
 
     Boundary startBoundary;
+
+
     @BeforeEach
     void setUp() {
+
         startBoundary = new Boundary(10, 100);
     }
 
 
     @Test
     void testContains() {
+
         assertTrue(startBoundary.contains(11));
         assertTrue(startBoundary.contains(50));
         assertFalse(startBoundary.contains(9));
@@ -34,6 +44,7 @@ class BoundaryTest {
 
     @Test
     void testIntersects() {
+
         assertTrue(startBoundary.intersects(new Boundary(1, 11)));
         assertTrue(startBoundary.intersects(new Boundary(11, 12)));
         assertTrue(startBoundary.intersects(new Boundary(11, 100)));
@@ -41,4 +52,17 @@ class BoundaryTest {
         assertTrue(startBoundary.intersects(new Boundary(99, 101)));
     }
 
+
+    @Test
+    void testSplit() {
+
+        assertEquals(4, startBoundary.split(List.of(12, 40, 90)).size());
+        assertEquals(List.of(new Boundary(10, 12), new Boundary(12, 40), new Boundary(40, 90), new Boundary(90, 100)), startBoundary.split(List.of(12, 40, 90)));
+        assertEquals(List.of(new Boundary(10, 40), new Boundary(40, 100)), startBoundary.split(List.of(40)));
+        assertEquals(1, startBoundary.split(Collections.emptyList()).size());
+        assertThrows(IndexOutOfBoundsException.class, () -> startBoundary.split(Collections.singletonList(0)));
+        assertThrows(IndexOutOfBoundsException.class, () -> startBoundary.split(Collections.singletonList(100)));
+        assertThrows(IndexOutOfBoundsException.class, () -> startBoundary.split(List.of(12, 40, 100)));
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java

@@ -6,9 +6,10 @@ import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
 import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
-import com.iqser.red.service.redaction.v1.server.document.services.DocumentDataMapper;
+import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentDataMapper;
 import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
-import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 
 import lombok.SneakyThrows;
@@ -40,7 +41,7 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
         var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
         DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
         DocumentData documentData = documentDataMapper.toDocumentData(document);
-        storageService.storeJSONObject(filename + ".json", documentData);
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + ".json", documentData);
         DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData);
 
         assert document.toString().equals(newDocumentGraph.toString());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java

@@ -16,7 +16,7 @@ import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
@@ -56,7 +56,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
 
         assertEquals("Expand to Hint ", entityNode.getTextBefore());
         assertEquals("’s Donut ←", entityNode.getTextAfter());
@@ -80,7 +80,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
 
         assertEquals("", entityNode.getTextBefore());
         assertEquals(" Purity Hint", entityNode.getTextAfter());
@@ -103,7 +103,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
 
         assertEquals("", entityNode.getTextBefore());
         assertEquals("", entityNode.getTextAfter());
@@ -189,7 +189,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
 
         assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore());
         assertEquals(", Group 9;", entityNode.getTextAfter());
@@ -214,7 +214,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
 
         assertEquals("2.6.1 Summary of ", entityNode.getTextBefore());
         assertEquals(" and excretion in", entityNode.getTextAfter());
@@ -238,7 +238,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
 
         assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore());
         assertEquals(" of metabolite of", entityNode.getTextAfter());
@@ -293,14 +293,13 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
         assert start != -1;
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
-        EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
+        EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY,  documentGraph);
         PageNode pageNode = documentGraph.getPages().get(pageNumber - 1);
 
         assertEquals(entityNode.getValue(), searchTerm);
         assertTrue(pageNode.getEntities().contains(entityNode));
         assertTrue(documentGraph.getPages().stream().filter(page -> page != pageNode).noneMatch(page -> page.getEntities().contains(entityNode)));
         assertTrue(entityNode.getPages().contains(pageNode));
-        assertTrue(entityNode.getPositions().size() >= searchTerm.replace(" ", "").length());
         assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode);
         assertTrue(entityNode.getIntersectingNodes().stream().allMatch(node -> node.getEntities().contains(entityNode)));
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java

@@ -17,8 +17,8 @@ public class EntitySearchUtilsTest {
     public void testNestedEntitiesRemoval() {
 
         Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         entities.add(nested);
         entities.add(nesting);
         EntitySearchUtils.removeEntitiesContainedInLarger(entities);
@@ -40,14 +40,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -73,14 +73,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -105,14 +105,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -137,15 +137,15 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
         foundEntities.add(foundEntities3);
@@ -171,16 +171,16 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntitiesOverlap1);
         foundEntities.add(foundEntitiesOverlap2);
         foundEntities.add(foundEntitiesSubset1);

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl

@@ -0,0 +1,56 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch;
+
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.*
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import java.util.Set;
+
+global DocumentGraph document
+
+rule "remove contained entity"
+
+    when
+        $largerEntity: EntityNode()
+        $smallerEntity: EntityNode(this != $largerEntity, containedBy($largerEntity))
+    then
+        $smallerEntity.removeFromGraph();
+        $smallerEntity.setRemoved(true);
+        $smallerEntity.setSkipRemoveEntitiesContainedInLarger(true);
+        System.out.printf("%s contained by %s\n", $smallerEntity, $largerEntity);
+        delete($smallerEntity);
+     end
+
+
+
+
+rule "1: Redact CBI_author"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        entity: EntityNode(type == "CBI_author")
+    then
+        entity.setRedaction(true);
+        update(entity)
+    end
+
+rule "2: do not redact genitive CBI_author"
+
+    when
+        entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
+    then
+        entity.setRedaction(false);
+        entity.setEntityType(EntityType.FALSE_POSITIVE);
+        update(entity)
+    end
+
+