RED-6009: Document Tree Structure

*renamed old TextBlock and Table Classes
*added subsections to enable separate rules for tables and text-only sections
*added ManualRedaction Tests
*finished default rules

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-server-v1</artifactId>
 
     <properties>
-        <drools.version>7.73.0.Final</drools.version>
+        <drools.version>8.37.0.Final</drools.version>
         <kie.version>7.73.0.Final</kie.version>
         <locationtech.version>1.19.0</locationtech.version>
         <javaassist.version>3.29.2-GA</javaassist.version>
@@ -64,7 +64,12 @@
         </dependency>
         <dependency>
             <groupId>org.drools</groupId>
-            <artifactId>drools-core</artifactId>
+            <artifactId>drools-engine</artifactId>
+            <version>${drools.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.drools</groupId>
+            <artifactId>drools-mvel</artifactId>
             <version>${drools.version}</version>
         </dependency>
         <dependency>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -33,7 +33,7 @@ public abstract class AbstractPageBlock {
     public abstract String getText();
 
 
-    public boolean containsBlock(TextBlock other) {
+    public boolean containsBlock(ClassificationTextBlock other) {
 
         return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Footer.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.AllArgsConstructor;
@@ -13,7 +13,7 @@ import lombok.Data;
 @AllArgsConstructor
 public class Footer {
 
-    private List<TextBlock> textBlocks;
+    private List<ClassificationTextBlock> textBlocks;
 
 
     @JsonIgnore

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Header.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.AllArgsConstructor;
@@ -13,7 +13,7 @@ import lombok.Data;
 @AllArgsConstructor
 public class Header {
 
-    private List<TextBlock> textBlocks;
+    private List<ClassificationTextBlock> textBlocks;
 
 
     @JsonIgnore

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/Section.java

@@ -4,8 +4,8 @@ import java.util.ArrayList;
 import java.util.List;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.Data;
@@ -24,32 +24,32 @@ public class Section implements Comparable {
 
         SearchableText searchableText = new SearchableText();
         pageBlocks.forEach(block -> {
-            if (block instanceof TextBlock) {
+            if (block instanceof ClassificationTextBlock) {
-                searchableText.addAll(((TextBlock) block).getSequences());
+                searchableText.addAll(((ClassificationTextBlock) block).getSequences());
             }
         });
         return searchableText;
     }
 
 
-    public List<Table> getTables() {
+    public List<TablePageBlock> getTables() {
 
-        List<Table> tables = new ArrayList<>();
+        List<TablePageBlock> tables = new ArrayList<>();
         pageBlocks.forEach(block -> {
-            if (block instanceof Table) {
+            if (block instanceof TablePageBlock) {
-                tables.add((Table) block);
+                tables.add((TablePageBlock) block);
             }
         });
         return tables;
     }
 
 
-    public List<TextBlock> getTextBlocks() {
+    public List<ClassificationTextBlock> getTextBlocks() {
 
-        List<TextBlock> textBlocks = new ArrayList<>();
+        List<ClassificationTextBlock> textBlocks = new ArrayList<>();
         pageBlocks.forEach(block -> {
-            if (block instanceof TextBlock) {
+            if (block instanceof ClassificationTextBlock) {
-                textBlocks.add((TextBlock) block);
+                textBlocks.add((ClassificationTextBlock) block);
             }
         });
         return textBlocks;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Cell.java

@@ -5,7 +5,7 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
@@ -19,7 +19,7 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public class Cell extends Rectangle {
 
-    private List<TextBlock> textBlocks = new ArrayList<>();
+    private List<ClassificationTextBlock> textBlocks = new ArrayList<>();
 
     private List<Cell> headerCells = new ArrayList<>();
 
@@ -32,7 +32,7 @@ public class Cell extends Rectangle {
     }
 
 
-    public void addTextBlock(TextBlock textBlock) {
+    public void addTextBlock(ClassificationTextBlock textBlock) {
 
         textBlocks.add(textBlock);
     }
@@ -43,11 +43,11 @@ public class Cell extends Rectangle {
 
         StringBuilder sb = new StringBuilder();
 
-        Iterator<TextBlock> itty = textBlocks.iterator();
+        Iterator<ClassificationTextBlock> itty = textBlocks.iterator();
         TextPositionSequence previous = null;
         while (itty.hasNext()) {
 
-            TextBlock textBlock = itty.next();
+            ClassificationTextBlock textBlock = itty.next();
 
             for (TextPositionSequence word : textBlock.getSequences()) {
                 if (previous != null) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java

@@ -12,14 +12,14 @@ import java.util.stream.Collectors;
 import org.apache.commons.collections4.CollectionUtils;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-public class Table extends AbstractPageBlock {
+public class TablePageBlock extends AbstractPageBlock {
 
     private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();
 
@@ -34,14 +34,14 @@ public class Table extends AbstractPageBlock {
     private List<List<Cell>> rows;
 
 
-    public Table(List<Cell> cells, Rectangle area, int rotation) {
+    public TablePageBlock(List<Cell> cells, Rectangle area, int rotation) {
 
         addCells(cells);
         minX = area.getLeft();
         minY = area.getBottom();
         maxX = area.getRight();
         maxY = area.getTop();
-        classification = "Table";
+        classification = "TablePageBlock";
         this.rotation = rotation;
 
     }
@@ -224,7 +224,7 @@ public class Table extends AbstractPageBlock {
      * Calculates the structure of the table. For spanning rows and columns multiple cells with the same values will be inserted.
      *
      * @param cells The found cells
-     * @return Table Structure
+     * @return TablePageBlock Structure
      */
     private List<List<Cell>> calculateStructure(List<Cell> cells) {
 
@@ -297,7 +297,7 @@ public class Table extends AbstractPageBlock {
                     }
                     if (column != null && column.getTextBlocks() != null) {
                         boolean first = true;
-                        for (TextBlock textBlock : column.getTextBlocks()) {
+                        for (ClassificationTextBlock textBlock : column.getTextBlocks()) {
                             if (!first) {
                                 sb.append("\n");
                             }
@@ -329,7 +329,7 @@ public class Table extends AbstractPageBlock {
                     sb.append(i == 0 ? "\n<th>" : "\n<td>");
                     if (column != null && column.getTextBlocks() != null) {
                         boolean first = true;
-                        for (TextBlock textBlock : column.getTextBlocks()) {
+                        for (ClassificationTextBlock textBlock : column.getTextBlocks()) {
                             if (!first) {
                                 sb.append("<br />");
                             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java

@@ -16,7 +16,7 @@ import lombok.NoArgsConstructor;
 @Builder
 @Data
 @NoArgsConstructor
-public class TextBlock extends AbstractPageBlock {
+public class ClassificationTextBlock extends AbstractPageBlock {
 
     @Builder.Default
     private List<TextPositionSequence> sequences = new ArrayList<>();
@@ -175,7 +175,7 @@ public class TextBlock extends AbstractPageBlock {
     }
 
 
-    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) {
+    public ClassificationTextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) {
 
         this.indexOnPage = indexOnPage;
         this.minX = minX;
@@ -187,23 +187,23 @@ public class TextBlock extends AbstractPageBlock {
     }
 
 
-    public TextBlock union(TextPositionSequence r) {
+    public ClassificationTextBlock union(TextPositionSequence r) {
 
-        TextBlock union = this.copy();
+        ClassificationTextBlock union = this.copy();
         union.add(r);
         return union;
     }
 
 
-    public TextBlock union(TextBlock r) {
+    public ClassificationTextBlock union(ClassificationTextBlock r) {
 
-        TextBlock union = this.copy();
+        ClassificationTextBlock union = this.copy();
         union.add(r);
         return union;
     }
 
 
-    public void add(TextBlock r) {
+    public void add(ClassificationTextBlock r) {
 
         if (r.getMinX() < minX) {
             minX = r.getMinX();
@@ -238,9 +238,9 @@ public class TextBlock extends AbstractPageBlock {
     }
 
 
-    public TextBlock copy() {
+    public ClassificationTextBlock copy() {
 
-        return new TextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage);
+        return new ClassificationTextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SectionText.java

@@ -35,7 +35,7 @@ public class SectionText {
     @Builder.Default
     private Set<Image> images = new HashSet<>();
     @Builder.Default
-    private List<TextBlock> textBlocks = new ArrayList<>();
+    private List<ClassificationTextBlock> textBlocks = new ArrayList<>();
     @Builder.Default
     private Map<String, CellValue> tabularData = new HashMap<>();
     @Builder.Default

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java

@@ -12,7 +12,7 @@ import lombok.Data;
 @AllArgsConstructor
 public class UnclassifiedText {
 
-    private List<TextBlock> textBlocks;
+    private List<ClassificationTextBlock> textBlocks;
 
 
     @JsonIgnore

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java

@@ -14,8 +14,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.RulingTextDirAdjustUtil;
 
@@ -29,7 +29,7 @@ public class BlockificationService {
     /**
      * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
      * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
-     * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
+     * Rulings (TablePageBlock lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
      *
      * @param textPositions         The words of a page.
      * @param horizontalRulingLines Horizontal table lines.
@@ -64,7 +64,7 @@ public class BlockificationService {
                     prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
                 }
 
-                TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+                ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
                 indexOnPage++;
 
                 chunkBlockList1.add(cb1);
@@ -106,17 +106,17 @@ public class BlockificationService {
             }
         }
 
-        TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+        ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
         if (cb1 != null) {
             chunkBlockList1.add(cb1);
         }
 
         Iterator<AbstractPageBlock> itty = chunkBlockList1.iterator();
 
-        TextBlock previousLeft = null;
+        ClassificationTextBlock previousLeft = null;
-        TextBlock previousRight = null;
+        ClassificationTextBlock previousRight = null;
         while (itty.hasNext()) {
-            TextBlock block = (TextBlock) itty.next();
+            ClassificationTextBlock block = (ClassificationTextBlock) itty.next();
 
             if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
                 if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
@@ -142,9 +142,9 @@ public class BlockificationService {
         }
 
         itty = chunkBlockList1.iterator();
-        TextBlock previous = null;
+        ClassificationTextBlock previous = null;
         while (itty.hasNext()) {
-            TextBlock block = (TextBlock) itty.next();
+            ClassificationTextBlock block = (ClassificationTextBlock) itty.next();
 
             if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
                     previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
@@ -167,9 +167,9 @@ public class BlockificationService {
     }
 
 
-    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    private ClassificationTextBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
-        TextBlock textBlock = null;
+        ClassificationTextBlock textBlock = null;
 
         FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
         FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
@@ -186,7 +186,7 @@ public class BlockificationService {
             styleFrequencyCounter.add(wordBlock.getFontStyle());
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinXDirAdj(),
+                textBlock = new ClassificationTextBlock(wordBlock.getMinXDirAdj(),
                         wordBlock.getMaxXDirAdj(),
                         wordBlock.getMinYDirAdj(),
                         wordBlock.getMaxYDirAdj(),
@@ -194,7 +194,7 @@ public class BlockificationService {
                         wordBlock.getRotation(),
                         indexOnPage);
             } else {
-                TextBlock spatialEntity = textBlock.union(wordBlock);
+                ClassificationTextBlock spatialEntity = textBlock.union(wordBlock);
                 textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java

@@ -10,8 +10,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
 
 @Service
@@ -77,8 +77,8 @@ public class BodyTextFrameService {
 
             for (AbstractPageBlock container : page.getTextBlocks()) {
 
-                if (container instanceof TextBlock) {
+                if (container instanceof ClassificationTextBlock) {
-                    TextBlock textBlock = (TextBlock) container;
+                    ClassificationTextBlock textBlock = (ClassificationTextBlock) container;
                     if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
                         continue;
                     }
@@ -94,15 +94,15 @@ public class BodyTextFrameService {
                     }
                 }
 
-                if (container instanceof Table) {
+                if (container instanceof TablePageBlock) {
-                    Table table = (Table) container;
+                    TablePageBlock table = (TablePageBlock) container;
                     for (List<Cell> row : table.getRows()) {
                         for (Cell cell : row) {
 
                             if (cell == null || cell.getTextBlocks() == null) {
                                 continue;
                             }
-                            for (TextBlock textBlock : cell.getTextBlocks()) {
+                            for (ClassificationTextBlock textBlock : cell.getTextBlocks()) {
                                 expandRectangle(textBlock, page, expansionsRectangle);
                             }
                         }
@@ -117,7 +117,7 @@ public class BodyTextFrameService {
     }
 
 
-    private void expandRectangle(TextBlock textBlock, Page page, BodyTextFrameExpansionsRectangle expansionsRectangle) {
+    private void expandRectangle(ClassificationTextBlock textBlock, Page page, BodyTextFrameExpansionsRectangle expansionsRectangle) {
 
         if (page.getPageWidth() > page.getPageHeight() && page.getRotation() != 0) {
             if (textBlock.getPdfMinY() < expansionsRectangle.minX) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java

@@ -9,7 +9,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
 
 import lombok.RequiredArgsConstructor;
@@ -41,14 +41,14 @@ public class ClassificationService {
     public void classifyPage(Page page, Document document, List<Float> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
-            if (textBlock instanceof TextBlock) {
+            if (textBlock instanceof ClassificationTextBlock) {
-                classifyBlock((TextBlock) textBlock, page, document, headlineFontSizes);
+                classifyBlock((ClassificationTextBlock) textBlock, page, document, headlineFontSizes);
             }
         }
     }
 
 
-    public void classifyBlock(TextBlock textBlock, Page page, Document document, List<Float> headlineFontSizes) {
+    public void classifyBlock(ClassificationTextBlock textBlock, Page page, Document document, List<Float> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
@@ -84,7 +84,7 @@ public class ClassificationService {
                     document.setHeadlines(true);
                 }
             }
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+        } else if (!textBlock.getText().startsWith("TablePageBlock ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
                 textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
                 .getMostPopular()
                 .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java

@@ -23,7 +23,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing.PDFLinesTextStripper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.FileUtils;
@@ -154,11 +154,11 @@ public class PdfSegmentationService {
 
         // Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame.
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
-            if (textBlock instanceof TextBlock) {
+            if (textBlock instanceof ClassificationTextBlock) {
-                if (((TextBlock) textBlock).getSequences() == null) {
+                if (((ClassificationTextBlock) textBlock).getSequences() == null) {
                     continue;
                 }
-                for (TextPositionSequence word : ((TextBlock) textBlock).getSequences()) {
+                for (TextPositionSequence word : ((ClassificationTextBlock) textBlock).getSequences()) {
                     page.getTextHeightCounter().add(word.getTextHeight());
                     page.getFontCounter().add(word.getFont());
                     page.getFontSizeCounter().add(word.getFontSize());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java

@@ -18,8 +18,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
 
 import lombok.extern.slf4j.Slf4j;
@@ -39,11 +39,11 @@ public class SectionsBuilderService {
         AbstractPageBlock prev = null;
 
         String lastHeadline = "";
-        Table previousTable = null;
+        TablePageBlock previousTable = null;
         for (Page page : document.getPages()) {
-            List<TextBlock> header = new ArrayList<>();
+            List<ClassificationTextBlock> header = new ArrayList<>();
-            List<TextBlock> footer = new ArrayList<>();
+            List<ClassificationTextBlock> footer = new ArrayList<>();
-            List<TextBlock> unclassifiedText = new ArrayList<>();
+            List<ClassificationTextBlock> unclassifiedText = new ArrayList<>();
             for (AbstractPageBlock current : page.getTextBlocks()) {
 
                 if (current.getClassification() == null) {
@@ -53,17 +53,17 @@ public class SectionsBuilderService {
                 current.setPage(page.getPageNumber());
 
                 if (current.getClassification().equals("Header")) {
-                    header.add((TextBlock) current);
+                    header.add((ClassificationTextBlock) current);
                     continue;
                 }
 
                 if (current.getClassification().equals("Footer")) {
-                    footer.add((TextBlock) current);
+                    footer.add((ClassificationTextBlock) current);
                     continue;
                 }
 
                 if (current.getClassification().equals("Other")) {
-                    unclassifiedText.add((TextBlock) current);
+                    unclassifiedText.add((ClassificationTextBlock) current);
                     continue;
                 }
 
@@ -79,7 +79,7 @@ public class SectionsBuilderService {
                         previousTable = chunkBlock.getTables().get(chunkBlock.getTables().size() - 1);
                     }
                 }
-                if (current instanceof Table table) {
+                if (current instanceof TablePageBlock table) {
                     // Distribute header information for subsequent tables
                     mergeTableMetadata(table, previousTable);
                     previousTable = table;
@@ -212,7 +212,7 @@ public class SectionsBuilderService {
     }
 
 
-    private void mergeTableMetadata(Table currentTable, Table previousTable) {
+    private void mergeTableMetadata(TablePageBlock currentTable, TablePageBlock previousTable) {
 
         // Distribute header information for subsequent tables
         if (previousTable != null && hasInvalidHeaderInformation(currentTable) && hasValidHeaderInformation(previousTable)) {
@@ -245,39 +245,39 @@ public class SectionsBuilderService {
         Section section = new Section();
 
         for (AbstractPageBlock container : wordBlockList) {
-            if (container instanceof Table table) {
+            if (container instanceof TablePageBlock table) {
 
                 if (lastHeadline == null || lastHeadline.isEmpty()) {
                     table.setHeadline("Text in table");
                 } else {
-                    table.setHeadline("Table in: " + lastHeadline);
+                    table.setHeadline("TablePageBlock in: " + lastHeadline);
                 }
 
                 section.getPageBlocks().add(table);
                 continue;
             }
 
-            TextBlock wordBlock = (TextBlock) container;
+            ClassificationTextBlock wordBlock = (ClassificationTextBlock) container;
             section.getPageBlocks().add(wordBlock);
         }
         return section;
     }
 
 
-    private boolean hasValidHeaderInformation(Table table) {
+    private boolean hasValidHeaderInformation(TablePageBlock table) {
 
         return !hasInvalidHeaderInformation(table);
     }
 
 
-    private boolean hasInvalidHeaderInformation(Table table) {
+    private boolean hasInvalidHeaderInformation(TablePageBlock table) {
 
         return table.getRows().stream().flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells()))).findAny().isEmpty();
 
     }
 
 
-    private List<Cell> getRowWithNonHeaderCells(Table table) {
+    private List<Cell> getRowWithNonHeaderCells(TablePageBlock table) {
 
         for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
             List<Cell> row = table.getRows().get(i);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java

@@ -19,8 +19,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Rectangle;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
 
 @Service
@@ -83,10 +83,10 @@ public class TableExtractionService {
 
         List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
 
-        List<TextBlock> toBeRemoved = new ArrayList<>();
+        List<ClassificationTextBlock> toBeRemoved = new ArrayList<>();
 
         for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
-            TextBlock textBlock = (TextBlock) abstractPageBlock;
+            ClassificationTextBlock textBlock = (ClassificationTextBlock) abstractPageBlock;
             for (Cell cell : cells) {
                 if (cell.intersects(textBlock.getPdfMinX(),
                         textBlock.getPdfMinY(),
@@ -104,7 +104,7 @@ public class TableExtractionService {
 
         List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).collect(Collectors.toList());
 
-        List<Table> tables = new ArrayList<>();
+        List<TablePageBlock> tables = new ArrayList<>();
         for (Rectangle area : spreadsheetAreas) {
 
             List<Cell> overlappingCells = new ArrayList<>();
@@ -113,16 +113,16 @@ public class TableExtractionService {
                     overlappingCells.add(c);
                 }
             }
-            tables.add(new Table(overlappingCells, area, page.getRotation()));
+            tables.add(new TablePageBlock(overlappingCells, area, page.getRotation()));
         }
 
-        for (Table table : tables) {
+        for (TablePageBlock table : tables) {
             int position = -1;
 
             Iterator<AbstractPageBlock> itty = page.getTextBlocks().iterator();
             while (itty.hasNext()) {
                 AbstractPageBlock textBlock = itty.next();
-                if (textBlock instanceof TextBlock ? table.containsBlock((TextBlock) textBlock) : table.contains(textBlock) && position == -1) {
+                if (textBlock instanceof ClassificationTextBlock ? table.containsBlock((ClassificationTextBlock) textBlock) : table.contains(textBlock) && position == -1) {
                     position = page.getTextBlocks().indexOf(textBlock);
                 }
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 
 import lombok.experimental.UtilityClass;
 
@@ -11,7 +11,7 @@ public final class PositionUtils {
 
     // TODO This currently uses pdf coord system. In the futher this should use java coord system.
     // Note: DirAdj (TextDirection Adjusted) can not be user for this.
-    public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) {
+    public boolean isWithinBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock) {
 
         if (btf == null || textBlock == null) {
             return false;
@@ -32,7 +32,7 @@ public final class PositionUtils {
 
     // TODO This currently uses pdf coord system. In the futher this should use java coord system.
     // Note: DirAdj (TextDirection Adjusted) can not be user for this.
-    public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
+    public boolean isOverBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock, int rotation) {
 
         if (btf == null || textBlock == null) {
             return false;
@@ -61,7 +61,7 @@ public final class PositionUtils {
 
     // TODO This currently uses pdf coord system. In the futher this should use java coord system.
     // Note: DirAdj (TextDirection Adjusted) can not be user for this.
-    public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
+    public boolean isUnderBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock, int rotation) {
 
         if (btf == null || textBlock == null) {
             return false;
@@ -90,7 +90,7 @@ public final class PositionUtils {
 
     // TODO This currently uses pdf coord system. In the futher this should use java coord system.
     // Note: DirAdj (TextDirection Adjusted) can not be user for this.
-    public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {
+    public boolean isTouchingUnderBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock) {
 
         //TODO Currently this is not working for rotated pages.
 
@@ -107,13 +107,13 @@ public final class PositionUtils {
     }
 
 
-    public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) {
+    public float getHeightDifferenceBetweenChunkWordAndDocumentWord(ClassificationTextBlock textBlock, Float documentMostPopularWordHeight) {
 
         return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
     }
 
 
-    public Float getApproxLineCount(TextBlock textBlock) {
+    public Float getApproxLineCount(ClassificationTextBlock textBlock) {
 
         return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java

@@ -10,10 +10,11 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.Ato
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
@@ -69,6 +70,7 @@ public class DocumentDataMapper {
             case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.getNode());
             case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.getNode());
             case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.getNode());
+            case SECTION -> PropertiesMapper.buildSectionProperties((SectionNode) entry.getNode());
             default -> new HashMap<>();
         };
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java

@@ -18,8 +18,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.Doc
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeaderNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
@@ -71,7 +71,7 @@ public class DocumentGraphMapper {
             List<PageNode> pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList();
 
             SemanticNode node = switch (entryData.getType()) {
-                case SECTION -> buildSection(context);
+                case SECTION -> buildSection(context, entryData.getProperties());
                 case PARAGRAPH -> buildParagraph(context, terminal);
                 case HEADLINE -> buildHeadline(context, terminal);
                 case HEADER -> buildHeader(context, terminal);
@@ -134,7 +134,7 @@ public class DocumentGraphMapper {
 
         TableNode.TableNodeBuilder builder = TableNode.builder();
         PropertiesMapper.parseTableProperties(properties, builder);
-        return TableNode.builder().tableOfContents(context.tableOfContents()).build();
+        return builder.tableOfContents(context.tableOfContents()).build();
     }
 
 
@@ -150,10 +150,11 @@ public class DocumentGraphMapper {
     }
 
 
-    private SectionNode buildSection(Context context) {
+    private SectionNode buildSection(Context context, Map<String, String> properties) {
-
-        return SectionNode.builder().tableOfContents(context.tableOfContents()).build();
 
+        var builder = SectionNode.builder();
+        PropertiesMapper.parseSectionProperties(properties, builder);
+        return builder.tableOfContents(context.tableOfContents()).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java

@@ -6,6 +6,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
@@ -50,6 +51,14 @@ public class PropertiesMapper {
     }
 
 
+    public static Map<String, String> buildSectionProperties(SectionNode node) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put("excludesTables", String.valueOf(node.isExcludesTables()));
+        return properties;
+    }
+
+
     public static void parseImageProperties(Map<String, String> properties, ImageNode.ImageNodeBuilder builder) {
 
         builder.imageType(parseImageType(properties.get("imageType")));
@@ -75,6 +84,12 @@ public class PropertiesMapper {
     }
 
 
+    public static void parseSectionProperties(Map<String, String> properties, SectionNode.SectionNodeBuilder builder) {
+
+        builder.excludesTables(Boolean.parseBoolean(properties.get("excludesTables")));
+    }
+
+
     public static ImageType parseImageType(String imageType) {
 
         return switch (imageType) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java

@@ -1,13 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
 
 import static java.lang.String.format;
+import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toList;
 
 import java.awt.geom.Rectangle2D;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -15,8 +14,6 @@ import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Footer;
@@ -24,11 +21,11 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.FooterNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeaderNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
@@ -45,7 +42,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Re
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.TextPositionOperations;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 
-@Service
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
 public class DocumentGraphFactory {
 
     public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
@@ -57,7 +56,7 @@ public class DocumentGraphFactory {
         DocumentGraph documentGraph = new DocumentGraph();
         Context context = new Context(new TableOfContents(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
 
-        document.getPages().stream().map(this::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
+        document.getPages().stream().map(DocumentGraphFactory::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
         document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image));
         addSections(document, context);
         addHeaderAndFooterToEachPage(document, context);
@@ -72,68 +71,11 @@ public class DocumentGraphFactory {
 
     private void addSections(Document document, Context context) {
 
-        document.getSections().forEach(section -> addSection(null, section.getPageBlocks(), section.getImages(), context));
+        document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getPageBlocks(), section.getImages(), context));
     }
 
 
-    private void addSection(SemanticNode parentNode, List<AbstractPageBlock> pageBlocks, List<ClassifiedImage> images, Context context) {
+    public void addTable(SemanticNode parentNode, TablePageBlock table, Context context) {
-
-        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
-        SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
-
-        context.sections().add(sectionNode);
-        blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber));
-
-        List<Integer> tocId;
-        if (parentNode == null) {
-            tocId = context.tableOfContents.createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
-        } else {
-            tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
-        }
-        sectionNode.setTocId(tocId);
-        Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
-        for (AbstractPageBlock abstractPageBlock : pageBlocks) {
-
-            if (alreadyMerged.contains(abstractPageBlock)) {
-                continue;
-            }
-
-            if (abstractPageBlock instanceof TextBlock) {
-                List<TextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, pageBlocks);
-                alreadyMerged.addAll(textBlocks);
-                addParagraphOrHeadline(sectionNode, (TextBlock) abstractPageBlock, context, textBlocks);
-            }
-            if (abstractPageBlock instanceof Table) {
-                addTable(sectionNode, (Table) abstractPageBlock, context);
-            }
-        }
-        for (ClassifiedImage image : images) {
-
-            addImage(sectionNode, image, context);
-        }
-    }
-
-
-    private static List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
-
-        return pageBlocks.stream()
-                .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
-                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
-                .filter(abstractTextContainer -> abstractTextContainer instanceof TextBlock)
-                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
-                .map(abstractTextContainer -> (TextBlock) abstractTextContainer)
-                .toList();
-    }
-
-
-    private void addSectionNodeToPageNode(Context context, SectionNode sectionNode, Integer pageNumber) {
-
-        PageNode page = getPage(pageNumber, context);
-        page.getMainBody().add(sectionNode);
-    }
-
-
-    private void addTable(SemanticNode parentNode, Table table, Context context) {
 
         PageNode page = getPage(table.getPage(), context);
         TableNode tableNode = TableNode.builder().tableOfContents(context.tableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build();
@@ -151,11 +93,11 @@ public class DocumentGraphFactory {
     }
 
 
-    private void addTableCells(List<List<Cell>> rows, SemanticNode parentNode, Context context, int pageNumber) {
+    private void addTableCells(List<List<Cell>> rows, TableNode tableNode, Context context, int pageNumber) {
 
         for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
             for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, parentNode, pageNumber, context);
+                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, tableNode, pageNumber, context);
             }
         }
     }
@@ -190,7 +132,7 @@ public class DocumentGraphFactory {
             tableCellNode.setTerminal(true);
 
         } else if (firstTextBlockIsHeadline(cell)) {
-            addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), Collections.emptyList(), context);
+            SectionNodeFactory.addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
             tableCellNode.setTerminal(false);
 
         } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
@@ -220,18 +162,24 @@ public class DocumentGraphFactory {
     }
 
 
-    private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context) {
+    public static boolean pageBlockIsHeadline(AbstractPageBlock abstractPageBlock) {
 
-        addParagraphOrHeadline(parentNode, originalTextBlock, context, Collections.emptyList());
+        return abstractPageBlock instanceof ClassificationTextBlock && abstractPageBlock.getClassification() != null && abstractPageBlock.getClassification().startsWith("H");
     }
 
 
-    private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context, List<TextBlock> textBlocksToMerge) {
+    private void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context) {
+
+        addParagraphOrHeadline(parentNode, originalTextBlock, context, emptyList());
+    }
+
+
+    public void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context, List<ClassificationTextBlock> textBlocksToMerge) {
 
         PageNode page = getPage(originalTextBlock.getPage(), context);
 
         SemanticNode node;
-        if (originalTextBlock.getClassification() != null && originalTextBlock.getClassification().startsWith("H")) {
+        if (pageBlockIsHeadline(originalTextBlock)) {
             node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build();
         } else {
             node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build();
@@ -239,7 +187,7 @@ public class DocumentGraphFactory {
 
         page.getMainBody().add(node);
 
-        List<TextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
+        List<ClassificationTextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
         textBlocks.add(originalTextBlock);
         AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
 
@@ -256,7 +204,7 @@ public class DocumentGraphFactory {
     }
 
 
-    private void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
+    public void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
 
         Rectangle2D position = RectangleTransformations.toRectangle2D(image.getPosition());
         PageNode page = getPage(image.getPage(), context);
@@ -277,13 +225,13 @@ public class DocumentGraphFactory {
 
     private void addHeaderAndFooterToEachPage(Document document, Context context) {
 
-        Map<Integer, List<TextBlock>> headers = document.getHeaders()
+        Map<Integer, List<ClassificationTextBlock>> headers = document.getHeaders()
                 .stream()
                 .map(Header::getTextBlocks)
                 .flatMap(List::stream)
                 .collect(groupingBy(AbstractPageBlock::getPage, toList()));
 
-        Map<Integer, List<TextBlock>> footers = document.getFooters()
+        Map<Integer, List<ClassificationTextBlock>> footers = document.getFooters()
                 .stream()
                 .map(Footer::getTextBlocks)
                 .flatMap(List::stream)
@@ -307,7 +255,7 @@ public class DocumentGraphFactory {
     }
 
 
-    private void addFooter(List<TextBlock> textBlocks, Context context) {
+    private void addFooter(List<ClassificationTextBlock> textBlocks, Context context) {
 
         PageNode page = getPage(textBlocks.get(0).getPage(), context);
         FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
@@ -322,7 +270,7 @@ public class DocumentGraphFactory {
     }
 
 
-    public void addHeader(List<TextBlock> textBlocks, Context context) {
+    public void addHeader(List<ClassificationTextBlock> textBlocks, Context context) {
 
         PageNode page = getPage(textBlocks.get(0).getPage(), context);
         HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
@@ -374,7 +322,7 @@ public class DocumentGraphFactory {
     }
 
 
-    private PageNode getPage(int pageIndex, Context context) {
+    public PageNode getPage(int pageIndex, Context context) {
 
         return context.pages.keySet()
                 .stream()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java

@@ -0,0 +1,187 @@
+package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
+
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory.pageBlockIsHeadline;
+import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class SectionNodeFactory {
+
+    public void addSection(SemanticNode parentNode, List<AbstractPageBlock> pageBlocks, List<ClassifiedImage> images, DocumentGraphFactory.Context context) {
+
+        if (pageBlocks.isEmpty()) {
+            return;
+        }
+        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
+        SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
+
+        context.sections().add(sectionNode);
+        blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber));
+
+        List<Integer> tocId;
+        if (parentNode == null) {
+            tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
+        } else {
+            tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
+        }
+        sectionNode.setTocId(tocId);
+
+        addFirstHeadlineDirectlyToSection(pageBlocks, context, sectionNode);
+        if (containsTablesAndTextBlocks(pageBlocks)) {
+            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(sectionNode, subSectionPageBlocks, emptyList(), context));
+        } else {
+            addTablesOrParagraphsOrHeadlinesToSection(pageBlocks, context, sectionNode);
+            if (listIsTextBlockOnly(pageBlocks)) {
+                sectionNode.setExcludesTables(true);
+            }
+        }
+
+        for (ClassifiedImage image : images) {
+            DocumentGraphFactory.addImage(sectionNode, image, context);
+        }
+    }
+
+
+    private static boolean listIsTextBlockOnly(List<AbstractPageBlock> pageBlocks) {
+
+        return pageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof ClassificationTextBlock);
+    }
+
+
+    private void addFirstHeadlineDirectlyToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, SectionNode sectionNode) {
+
+        if (pageBlockIsHeadline(pageBlocks.get(0))) {
+            addTablesOrParagraphsOrHeadlinesToSection(List.of(pageBlocks.get(0)), context, sectionNode);
+            pageBlocks.remove(0);
+        }
+    }
+
+
+    private void addTablesOrParagraphsOrHeadlinesToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, SectionNode sectionNode) {
+
+        Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
+        for (AbstractPageBlock abstractPageBlock : pageBlocks) {
+
+            if (alreadyMerged.contains(abstractPageBlock)) {
+                continue;
+            }
+
+            if (abstractPageBlock instanceof ClassificationTextBlock) {
+                List<ClassificationTextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, pageBlocks);
+                alreadyMerged.addAll(textBlocks);
+                DocumentGraphFactory.addParagraphOrHeadline(sectionNode, (ClassificationTextBlock) abstractPageBlock, context, textBlocks);
+            }
+            if (abstractPageBlock instanceof TablePageBlock) {
+                DocumentGraphFactory.addTable(sectionNode, (TablePageBlock) abstractPageBlock, context);
+            }
+        }
+    }
+
+
+    private static boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {
+
+        return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
+                .anyMatch(pageBlock -> pageBlock instanceof ClassificationTextBlock);
+    }
+
+
+    /**
+     * This function splits the list of PageBlocks around TablePageBlocks, such that SubSections can be created, that don't include tables.
+     * First, the function splits the list into coherent Lists, which include only either type.
+     * If a Table is directly preceded by a Headline, it is moved to the Table list.
+     *
+     * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock
+     * @return List of Lists of AbstractPageBlocks, which include either a single Headline ClassificationTextBlock and a TablePageBlock or only ClassificationTextBlocks.
+     */
+    private List<List<AbstractPageBlock>> splitPageBlocksIntoSubSections(List<AbstractPageBlock> pageBlocks) {
+
+        List<List<AbstractPageBlock>> splitList = splitIntoCoherentList(pageBlocks);
+        movePrecedingHeadlineToTableList(splitList);
+        return splitList.stream().filter(list -> !list.isEmpty()).toList();
+    }
+
+
+    private static void movePrecedingHeadlineToTableList(List<List<AbstractPageBlock>> splitList) {
+
+        for (int i = 0; i < splitList.size(); i++) {
+            if (listIsTableOnly(splitList.get(i))) {
+                if (i > 0) {
+                    List<AbstractPageBlock> previousList = splitList.get(i - 1);
+                    AbstractPageBlock lastPageBlockInPreviousList = previousList.get(previousList.size() - 1);
+                    if (pageBlockIsHeadline(lastPageBlockInPreviousList)) {
+                        previousList.remove(i - 1);
+                        splitList.get(i).add(0, lastPageBlockInPreviousList);
+                    }
+                }
+            }
+        }
+    }
+
+
+    private static boolean listIsTableOnly(List<AbstractPageBlock> abstractPageBlocks) {
+
+        return abstractPageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock);
+    }
+
+
+    /**
+     * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock
+     * @return List of Lists of AbstractPageBlocks, which are exclusively of type ClassificationTextBlock or TablePageBlock
+     */
+    private static List<List<AbstractPageBlock>> splitIntoCoherentList(List<AbstractPageBlock> pageBlocks) {
+
+        List<List<AbstractPageBlock>> splitList = new LinkedList<>();
+        List<AbstractPageBlock> currentList = new LinkedList<>();
+        splitList.add(currentList);
+
+        Class<? extends AbstractPageBlock> lastPageBlockClass = pageBlocks.get(0).getClass();
+        for (AbstractPageBlock pageBlock : pageBlocks) {
+            if (lastPageBlockClass.isInstance(pageBlock)) {
+                currentList.add(pageBlock);
+            } else {
+                currentList = new LinkedList<>();
+                currentList.add(pageBlock);
+                splitList.add(currentList);
+                lastPageBlockClass = pageBlock.getClass();
+            }
+        }
+        return splitList;
+    }
+
+
+    private static List<ClassificationTextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
+
+        return pageBlocks.stream()
+                .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
+                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
+                .filter(abstractTextContainer -> abstractTextContainer instanceof ClassificationTextBlock)
+                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
+                .map(abstractTextContainer -> (ClassificationTextBlock) abstractTextContainer)
+                .toList();
+    }
+
+
+    private void addSectionNodeToPageNode(DocumentGraphFactory.Context context, SectionNode sectionNode, Integer pageNumber) {
+
+        PageNode page = DocumentGraphFactory.getPage(pageNumber, context);
+        page.getMainBody().add(sectionNode);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
 
 import static java.lang.String.format;
 
+import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -105,7 +106,7 @@ public class Boundary implements Comparable<Boundary> {
     }
 
 
-    public static Boundary merge(List<Boundary> boundaries) {
+    public static Boundary merge(Collection<Boundary> boundaries) {
 
         int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
         int maxEnd = boundaries.stream().mapToInt(Boundary::end).max().orElseThrow(IllegalArgumentException::new);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/TableOfContents.java

@@ -9,6 +9,7 @@ import java.util.List;
 import java.util.stream.Stream;
 
 import com.google.common.hash.Hashing;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
@@ -84,7 +85,7 @@ public class TableOfContents {
 
     public boolean hasParentById(List<Integer> tocId) {
 
-        return entryExists(getParentId(tocId));
+        return !tocId.isEmpty();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java

@@ -45,14 +45,15 @@ public class RedactionEntity {
     boolean dossierDictionaryEntry;
     Set<Engine> engines;
     Set<RedactionEntity> references;
-    int matchedRule;
+    @Builder.Default
+    List<Integer> matchedRules = new LinkedList<>();
     String redactionReason;
     String legalBasis;
 
     // inferred on graph insertion
     String value;
-    CharSequence textBefore;
+    String textBefore;
-    CharSequence textAfter;
+    String textAfter;
     @Builder.Default
     Set<PageNode> pages = new HashSet<>();
     List<RedactionPosition> redactionPositionsPerPage;
@@ -76,13 +77,30 @@ public class RedactionEntity {
                 .dossierDictionaryEntry(false)
                 .engines(new HashSet<>())
                 .references(new HashSet<>())
-                .matchedRule(-1)
                 .redactionReason("")
                 .legalBasis("")
                 .build();
     }
 
 
+    public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
+
+        return intersectingNodes.stream().anyMatch(clazz::isInstance);
+    }
+
+
+    public boolean isType(String type) {
+
+        return this.type.equals(type);
+    }
+
+
+    public boolean isAnyType(List<String> types) {
+
+        return types.contains(type);
+    }
+
+
     public void addIntersectingNode(SemanticNode containingNode) {
 
         intersectingNodes.add(containingNode);
@@ -91,10 +109,29 @@ public class RedactionEntity {
 
     public void removeFromGraph() {
 
-        getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
+        intersectingNodes.forEach(node -> node.getEntities().remove(this));
+        pages.forEach(page -> page.getEntities().remove(this));
+
+        intersectingNodes = new LinkedList<>();
         deepestFullyContainingNode = null;
-        getPages().forEach(page -> page.getEntities().remove(this));
+        pages = new HashSet<>();
-        setRemoved(true);
+        removed = true;
+    }
+
+
+    public void addMatchedRule(int ruleNumber) {
+
+        matchedRules.add(ruleNumber);
+        engines.add(Engine.RULE);
+    }
+
+
+    public int getMatchedRule() {
+
+        if (matchedRules.isEmpty()) {
+            return -1;
+        }
+        return matchedRules.get(matchedRules.size() - 1);
     }
 
 
@@ -155,9 +192,15 @@ public class RedactionEntity {
     }
 
 
-    public void addReference(RedactionEntity redactionEntity) {
+    public void addReference(RedactionEntity reference) {
 
-        references.add(redactionEntity);
+        references.add(reference);
+    }
+
+
+    public void addReferences(List<RedactionEntity> references) {
+
+        this.references.addAll(references);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/DocumentGraph.java

@@ -1,19 +1,18 @@
-package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
+package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
 
@@ -35,6 +34,8 @@ public class DocumentGraph implements SemanticNode {
     TableOfContents tableOfContents;
     Integer numberOfPages;
     TextBlock textBlock;
+    @Builder.Default
+    Set<RedactionEntity> entities = new HashSet<>();
 
 
     public TextBlock buildTextBlock() {
@@ -55,12 +56,6 @@ public class DocumentGraph implements SemanticNode {
     }
 
 
-    public Set<RedactionEntity> getEntities() {
-
-        return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
-    }
-
-
     @Override
     public List<Integer> getTocId() {
 
@@ -71,7 +66,17 @@ public class DocumentGraph implements SemanticNode {
     @Override
     public void setTocId(List<Integer> tocId) {
 
-        throw new UnsupportedOperationException("DocumentGraph is always the root of the Table of Contents");
+        throw new UnsupportedOperationException("DocumentGraph is always the root of the TablePageBlock of Contents");
+    }
+
+
+    @Override
+    public HeadlineNode getHeadline() {
+
+        return streamAllSubNodes().filter(node -> node instanceof HeadlineNode)
+                .map(node -> (HeadlineNode) node)
+                .findFirst()
+                .orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/HeadlineNode.java

@@ -52,7 +52,7 @@ public class HeadlineNode implements SemanticNode {
 
 
     @Override
-    public SemanticNode getHeadline() {
+    public HeadlineNode getHeadline() {
 
         return this;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionNode.java

@@ -4,7 +4,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
@@ -30,6 +29,7 @@ public class SectionNode implements SemanticNode {
     TextBlock textBlock;
     @EqualsAndHashCode.Exclude
     TableOfContents tableOfContents;
+    boolean excludesTables;
 
     @Builder.Default
     @EqualsAndHashCode.Exclude
@@ -55,10 +55,11 @@ public class SectionNode implements SemanticNode {
 
     public HeadlineNode getHeadline() {
 
-        return streamChildren().filter(node -> node instanceof HeadlineNode)
+        return streamChildren()//
-                .map(node -> (HeadlineNode) node)
+                .filter(node -> node instanceof HeadlineNode)//
-                .findFirst()
+                .map(node -> (HeadlineNode) node)//
-                .orElseThrow(() -> new NotFoundException("Section has no Headline!"));
+                .findFirst()//
+                .orElseGet(() -> getParent().getHeadline());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java

@@ -91,7 +91,7 @@ public interface SemanticNode {
      *
      * @return First HeadlineNode found
      */
-    default SemanticNode getHeadline() {
+    default HeadlineNode getHeadline() {
 
         return getParent().getHeadline();
     }
@@ -157,6 +157,24 @@ public interface SemanticNode {
     }
 
 
+    default boolean hasEntitiesOfType(String type) {
+
+        return getEntities().stream().anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
+    }
+
+
+    default List<RedactionEntity> getEntitiesOfType(String type) {
+
+        return getEntities().stream().filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
+    }
+
+
+    default List<RedactionEntity> getEntitiesOfType(List<String> types) {
+
+        return getEntities().stream().filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
+    }
+
+
     /**
      * Each AtomicTextBlock has an index on its page, this returns the number of the first AtomicTextBlock underneath this node.
      * If this node does not have any AtomicTexBlocks underneath it, e.g. an empty TableCell. It returns -1.
@@ -184,8 +202,8 @@ public interface SemanticNode {
 
 
     /**
-     * @param string A String which the ClassificationTextBlock might contain
+     * @param string A String which the TextBlock might contain
-     * @return true, if this node's ClassificationTextBlock contains the string
+     * @return true, if this node's TextBlock contains the string
      */
     default boolean containsString(String string) {
 
@@ -193,6 +211,26 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * @param strings A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains all strings
+     */
+    default boolean containsStrings(List<String> strings) {
+
+        return strings.stream().allMatch(this::containsString);
+    }
+
+
+    /**
+     * @param string A String which the ClassificationTextBlock might contain
+     * @return true, if this node's ClassificationTextBlock contains the string
+     */
+    default boolean containsStringIgnoreCase(String string) {
+
+        return buildTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
+    }
+
+
     /**
      * @param strings A List of Strings which the ClassificationTextBlock might contain
      * @return true, if this node's ClassificationTextBlock contains any of the strings
@@ -203,6 +241,16 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * @param strings A List of Strings which the ClassificationTextBlock might contain
+     * @return true, if this node's ClassificationTextBlock contains any of the strings
+     */
+    default boolean containsAnyStringIgnoreCase(List<String> strings) {
+
+        return strings.stream().anyMatch(this::containsStringIgnoreCase);
+    }
+
+
     /**
      * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the RedactionEntity intersects or even contains the RedactionEntity.
      * It sets the fields accordingly and recursively calls this function on all its children.
@@ -213,13 +261,13 @@ public interface SemanticNode {
 
         TextBlock textBlock = buildTextBlock();
         if (textBlock.getBoundary().intersects(redactionEntity.getBoundary())) {
-
             if (textBlock.containsBoundary(redactionEntity.getBoundary())) {
                 redactionEntity.setDeepestFullyContainingNode(this);
             }
 
             redactionEntity.addIntersectingNode(this);
-            streamChildren().forEach(node -> node.addThisToEntityIfIntersects(redactionEntity));
+            streamChildren().filter(semanticNode -> semanticNode.getBoundary().intersects(redactionEntity.getBoundary()))
+                    .forEach(node -> node.addThisToEntityIfIntersects(redactionEntity));
         }
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java

@@ -1,8 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
 
+import static java.lang.String.format;
+
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
@@ -26,8 +30,8 @@ public class TableNode implements SemanticNode {
     List<Integer> tocId;
     TableOfContents tableOfContents;
 
-    Integer numberOfRows;
+    int numberOfRows;
-    Integer numberOfCols;
+    int numberOfCols;
 
     TextBlock textBlock;
 
@@ -36,12 +40,95 @@ public class TableNode implements SemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    public Stream<RedactionEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
+
+        return IntStream.range(0, numberOfRows)
+                .boxed()
+                .filter(row -> rowContainsStringsIgnoreCase(row, strings))
+                .flatMap(this::streamRow)
+                .map(TableCellNode::getEntities)
+                .flatMap(Collection::stream);
+    }
+
+
+    public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
+
+        String rowText = streamRow(row).map(TableCellNode::buildTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase();
+        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
+    }
+
+
+    public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
+
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCellNode::getCol).toList();
+        return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
+                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
+                .map(TableCellNode::getEntities)
+                .flatMap(Collection::stream);
+    }
+
+
+    public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
+
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCellNode::getCol).toList();
+        return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
+                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsAnyString(values)))
+                .map(TableCellNode::getEntities)
+                .flatMap(Collection::stream);
+    }
+
+
+    public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(String type) {
+
+        List<Integer> rowsWithEntityOfType = getEntities().stream()
+                .filter(redactionEntity -> redactionEntity.getType().equals(type))
+                .map(RedactionEntity::getIntersectingNodes)
+                .filter(node -> node instanceof TableCellNode)
+                .map(node -> (TableCellNode) node)
+                .map(TableCellNode::getRow)
+                .toList();
+
+        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCellNode::getEntities).flatMap(Collection::stream);
+    }
+
+
+    public TableCellNode getCell(int row, int col) {
+
+        if (numberOfRows - row < 0 || numberOfCols - col < 0) {
+            throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
+        }
+        int idx = row * numberOfCols + col;
+        return (TableCellNode) tableOfContents.getEntryById(tocId).getChildren().get(idx).getNode();
+    }
+
+
     public Stream<TableCellNode> streamTableCells() {
 
         return streamChildren().map(node -> (TableCellNode) node);
     }
 
 
+    public Stream<TableCellNode> streamTableCellsWithHeader(String header) {
+
+        return streamHeaders().filter(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().contains(header))
+                .map(TableCellNode::getCol)
+                .flatMap(this::streamCol)
+                .filter(tableCellNode -> !tableCellNode.isHeader());
+    }
+
+
+    public Stream<TableCellNode> streamCol(int col) {
+
+        return streamTableCells().filter(tableCellNode -> tableCellNode.getCol() == col);
+    }
+
+
+    public Stream<TableCellNode> streamRow(int row) {
+
+        return streamTableCells().filter(tableCellNode -> tableCellNode.getRow() == row);
+    }
+
+
     public Stream<TableCellNode> streamHeaders() {
 
         return streamTableCells().filter(TableCellNode::isHeader);
@@ -54,6 +141,24 @@ public class TableNode implements SemanticNode {
     }
 
 
+    public boolean hasHeader(String header) {
+
+        return streamHeaders().anyMatch(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().contains(header));
+    }
+
+
+    public boolean hasRowWithHeaderAndValue(String header, String value) {
+
+        return streamTableCellsWithHeader(header).anyMatch(tableCellNode -> tableCellNode.containsString(value));
+    }
+
+
+    public boolean hasRowWithHeaderAndAnyValue(String header, List<String> values) {
+
+        return streamTableCellsWithHeader(header).anyMatch(tableCellNode -> tableCellNode.containsAnyString(values));
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 
@@ -67,7 +172,7 @@ public class TableNode implements SemanticNode {
     @Override
     public String toString() {
 
-        return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().buildSummary();
+        return tocId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + buildTextBlock().buildSummary();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java

@@ -7,6 +7,7 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
@@ -162,10 +163,7 @@ public class ConcatenatedTextBlock implements TextBlock {
     private Map<PageNode, List<Rectangle2D>> mergeEntityPositionsWithSamePageNode(Map<PageNode, List<Rectangle2D>> map1, Map<PageNode, List<Rectangle2D>> map2) {
 
         Map<PageNode, List<Rectangle2D>> mergedMap = new HashMap<>(map1);
-        map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, rectangles, (rects1, rects2) -> {
+        map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, rectangles, (l1, l2) -> Stream.concat(l1.stream(), l2.stream()).toList()));
-            rects1.addAll(rects2);
-            return rects1;
-        }));
         return mergedMap;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java

@@ -1,12 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
 
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.getExpandedEndByRegex;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.getExpandedStartByRegex;
 import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -16,12 +20,15 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.CharSequenceSearchUtils;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -37,8 +44,8 @@ public class EntityCreationService {
     public Set<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.buildTextBlock();
-        List<Boundary> startBoundaries = CharSequenceSearchUtils.findBoundariesByString(start, textBlock);
+        List<Boundary> startBoundaries = RedactionSearchUtils.findBoundariesByString(start, textBlock);
-        List<Boundary> stopBoundaries = CharSequenceSearchUtils.findBoundariesByString(stop, textBlock);
+        List<Boundary> stopBoundaries = RedactionSearchUtils.findBoundariesByString(stop, textBlock);
 
         List<Boundary> entityBoundaries = new LinkedList<>();
         if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
@@ -85,7 +92,7 @@ public class EntityCreationService {
     public Set<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.buildTextBlock();
-        List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByString(string, textBlock);
+        List<Boundary> boundaries = RedactionSearchUtils.findBoundariesByString(string, textBlock);
         return boundaries.stream()
                 .map(boundary -> toLineAfterBoundary(textBlock, boundary))
                 .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
@@ -96,7 +103,14 @@ public class EntityCreationService {
 
     public Set<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
 
-        List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
+        List<Boundary> boundaries = RedactionSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
+        return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
+    }
+
+
+    public Set<RedactionEntity> byKeyword(String keyword, String type, EntityType entityType, SemanticNode node) {
+
+        List<Boundary> boundaries = RedactionSearchUtils.findBoundariesByString(keyword, node.buildTextBlock());
         return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
     }
 
@@ -109,10 +123,24 @@ public class EntityCreationService {
     }
 
 
+    public RedactionEntity byPrefixExpansionRegex(RedactionEntity entity, String regexPattern) {
+
+        int expandedStart = getExpandedStartByRegex(entity, regexPattern);
+        return byBoundary(new Boundary(expandedStart, entity.getBoundary().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+    }
+
+
+    public RedactionEntity bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) {
+
+        int expandedEnd = getExpandedEndByRegex(entity, regexPattern);
+        return byBoundary(new Boundary(entity.getBoundary().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+    }
+
+
     public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
 
         RedactionEntity entity = RedactionEntity.initialEntityNode(boundary, type, entityType);
-        addEntityToGraph(entity, node.getTableOfContents());
+        addEntityToGraph(entity, node);
         return entity;
     }
 
@@ -129,25 +157,65 @@ public class EntityCreationService {
         RedactionEntity mergedEntity = RedactionEntity.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(RedactionEntity::getBoundary).toList()), type, entityType);
         mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(RedactionEntity::isRedaction));
         mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
+        entitiesToMerge.stream().map(RedactionEntity::getMatchedRules).flatMap(Collection::stream).forEach(mergedEntity::addMatchedRule);
+
         RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0));
         mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
-        mergedEntity.setMatchedRule(entityWithHigherRuleNumber.getMatchedRule());
         mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
 
-        addEntityToGraph(mergedEntity, node.getTableOfContents());
+        addEntityToGraph(mergedEntity, node);
         return mergedEntity;
     }
 
 
+    public RedactionEntity byTableCellAsHighlight(TableCellNode tableCellNode, String type, EntityType entityType) {
+
+        RedactionEntity highlightEntity = RedactionEntity.initialEntityNode(new Boundary(tableCellNode.getBoundary().start(), tableCellNode.getBoundary().start()),
+                type,
+                entityType);
+
+        String positionId = IdBuilder.buildId(tableCellNode.getBBox().keySet(), tableCellNode.getBBox().values().stream().toList());
+        highlightEntity.setRedactionPositionsPerPage(tableCellNode.getBBox()
+                .entrySet()
+                .stream()
+                .map(entry -> new RedactionPosition(positionId, entry.getKey(), List.of(entry.getValue())))
+                .toList());
+        addEntityToGraph(highlightEntity, tableCellNode);
+        return highlightEntity;
+    }
+
+
     public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
 
         return boundaryIsSurroundedBySeparators(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary);
     }
 
 
+    public void addEntityToGraph(RedactionEntity entity, SemanticNode node) {
+
+        TableOfContents tableOfContents = node.getTableOfContents();
+        try {
+            addEntityToGraph(entity, tableOfContents);
+            entity.addIntersectingNode(tableOfContents.getRoot().getNode());
+            tableOfContents.getRoot().getNode().getEntities().add(entity);
+        } catch (NoSuchElementException e) {
+            entity.setDeepestFullyContainingNode(tableOfContents.getRoot().getNode());
+            entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().buildTextBlock());
+            entity.addIntersectingNode(tableOfContents.getRoot().getNode());
+            addToPages(entity);
+            addEntityToNodeEntitySets(entity);
+        }
+    }
+
+
     public void addEntityToGraph(RedactionEntity entity, TableOfContents tableOfContents) {
 
-        tableOfContents.getRoot().getNode().addThisToEntityIfIntersects(entity);
+        SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList())
+                .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
+                .findFirst()
+                .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
+
+        containingNode.addThisToEntityIfIntersects(entity);
 
         TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock();
         entityEnrichmentService.enrichEntity(entity, textBlock);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
 
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@@ -29,7 +27,7 @@ public class EntityEnrichmentService {
     }
 
 
-    private CharSequence findTextAfter(int index, TextBlock textBlock) {
+    private String findTextAfter(int index, TextBlock textBlock) {
 
         int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().end());
         String textAfter = textBlock.subSequence(index, endOffset).toString();
@@ -44,7 +42,7 @@ public class EntityEnrichmentService {
     }
 
 
-    private CharSequence findTextBefore(int index, TextBlock textBlock) {
+    private String findTextBefore(int index, TextBlock textBlock) {
 
         int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().start());
         String textBefore = textBlock.subSequence(offsetBefore, index).toString();
@@ -90,27 +88,4 @@ public class EntityEnrichmentService {
         return startWithSpace ? " " + result : result;
     }
 
-
-    public static void setFields(RedactionEntity redactionEntity, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
-
-        if (redactionEntity.getMatchedRule() == -1 && matchedRule != -1) {
-            redactionEntity.setMatchedRule(matchedRule);
-        }
-        if (redactionEntity.getRedactionReason().equals("") && redactionReason != null) {
-            redactionEntity.setRedactionReason(redactionReason);
-        }
-        if (redactionEntity.getLegalBasis().equals("") && legalBasis != null) {
-            redactionEntity.setLegalBasis(legalBasis);
-        }
-        if (engine != null) {
-            redactionEntity.addEngine(engine);
-        }
-    }
-
-
-    public static void setFields(Collection<RedactionEntity> redactionEntities, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
-
-        redactionEntities.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java

@@ -7,6 +7,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
 
 import lombok.RequiredArgsConstructor;
@@ -22,7 +23,7 @@ public class ManualRedactionApplicationService {
 
         RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage()
                 .stream()
-                .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getValue()))
+                .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
                 .findFirst()
                 .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
 
@@ -38,8 +39,9 @@ public class ManualRedactionApplicationService {
         entityToBeResized.setResized(true);
         entityToBeResized.getBoundary().setStart(newStartOffset);
         entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
+        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getTableOfContents().getRoot().getNode();
         entityToBeResized.removeFromGraph();
-        entityCreationService.addEntityToGraph(entityToBeResized, entityToBeResized.getDeepestFullyContainingNode().getTableOfContents());
+        entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java

@@ -99,6 +99,16 @@ public class RectangleTransformations {
     }
 
 
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(),
+                (float) rectangle2D.getMaxY(),
+                (float) rectangle2D.getWidth(),
+                (float) rectangle2D.getHeight(),
+                pageNumber);
+    }
+
+
     public static String toString(Rectangle2D rectangle2D) {
 
         return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtils.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
 
-import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
+import static java.lang.String.format;
 
 import java.util.LinkedList;
 import java.util.List;
@@ -8,10 +8,11 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 
-public class CharSequenceSearchUtils {
+public class RedactionSearchUtils {
 
     public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
 
@@ -20,6 +21,12 @@ public class CharSequenceSearchUtils {
     }
 
 
+    public static boolean exactMatch(CharSequence charSequence, String regexPattern) {
+
+        return charSequence.toString().matches(regexPattern);
+    }
+
+
     public static boolean anyMatch(TextBlock textBlock, String regexPattern) {
 
         Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
@@ -31,8 +38,50 @@ public class CharSequenceSearchUtils {
 
         Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
         Matcher matcher = pattern.matcher(searchText);
+        if (matcher.find()) {
             return new Boundary(matcher.start(), matcher.end());
         }
+        throw new IllegalArgumentException(format("Charsequence %s does not contain any matches for pattern %s", searchText, regexPattern));
+    }
+
+
+    public static int getExpandedEndByRegex(RedactionEntity entity, String regexPattern) {
+
+        int expandedEnd;
+        if (anyMatch(entity.getTextAfter(), regexPattern)) {
+            Boundary postfixBoundary = findFirstBoundary(regexPattern, entity.getTextAfter());
+            expandedEnd = postfixBoundary.end() + entity.getBoundary().end();
+        } else {
+            expandedEnd = entity.getBoundary().end();
+        }
+        return expandedEnd;
+    }
+
+
+    public static int getExpandedStartByRegex(RedactionEntity entity, String regexPattern) {
+
+        int expandedStart;
+        if (anyMatch(entity.getTextBefore(), regexPattern)) {
+            Boundary prefixBoundary = findFirstBoundary(regexPattern, entity.getTextBefore());
+            expandedStart = prefixBoundary.start() + entity.getBoundary().start() - entity.getTextBefore().length();
+        } else {
+            expandedStart = entity.getBoundary().start();
+        }
+        return expandedStart;
+    }
+
+
+    public static List<Boundary> findBoundariesByRegex(String regexPattern, CharSequence searchText) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+        Matcher matcher = pattern.matcher(searchText);
+        List<Boundary> boundaries = new LinkedList<>();
+        while (matcher.find()) {
+            boundaries.add(new Boundary(matcher.start(), matcher.end()));
+        }
+        return boundaries;
+
+    }
 
 
     public static List<Boundary> findBoundariesByRegex(String regexPattern, TextBlock textBlock) {
@@ -43,7 +92,7 @@ public class CharSequenceSearchUtils {
         while (matcher.find()) {
             boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start()));
         }
-        return boundaries.stream().filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
+        return boundaries;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java

@@ -3,12 +3,12 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
 import java.util.Comparator;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 
 public class TextPositionOperations {
 
-    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<TextBlock> textBlocks) {
+    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<ClassificationTextBlock> textBlocks) {
 
         return textBlocks.stream()//
                 .flatMap(tb -> tb.getSequences().stream())//

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java

@@ -13,7 +13,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/conditions/SemanticNodeRuleConditions.java

@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class SemanticNodeRuleConditions {
+
+    public boolean hasEntitiesOfType(SemanticNode semanticNode, String type) {
+
+        return semanticNode.getEntities().stream().anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
+    }
+
+
+    public boolean hasEntitiesOfType(SemanticNode semanticNode, List<String> types) {
+
+        return types.stream().allMatch(type -> hasEntitiesOfType(semanticNode, type));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/CellValue.java

@@ -4,7 +4,7 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
@@ -17,7 +17,7 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class CellValue {
 
-    private List<TextBlock> textBlocks = new ArrayList<>();
+    private List<ClassificationTextBlock> textBlocks = new ArrayList<>();
 
     private int rowSpanStart;
 
@@ -27,9 +27,9 @@ public class CellValue {
 
         StringBuilder sb = new StringBuilder();
 
-        Iterator<TextBlock> itty = textBlocks.iterator();
+        Iterator<ClassificationTextBlock> itty = textBlocks.iterator();
         while (itty.hasNext()) {
-            TextBlock textBlock = itty.next();
+            ClassificationTextBlock textBlock = itty.next();
             TextPositionSequence previous = null;
             for (TextPositionSequence word : textBlock.getSequences()) {
                 if (previous != null) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -23,8 +23,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.RedTextPosition;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
@@ -1354,7 +1354,7 @@ public class Section {
             entity.setRedactionReason(reason);
             entity.setTargetSequences(value.getTextBlocks()
                     .stream()
-                    .map(TextBlock::getSequences)
+                    .map(ClassificationTextBlock::getSequences)
                     .flatMap(Collection::stream)
                     .collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
             entity.setLegalBasis(legalBasis);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -24,7 +24,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
@@ -36,7 +36,9 @@ import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @RequiredArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
@@ -103,8 +105,9 @@ public class DroolsExecutionService {
 
         kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
         kieSession.fireAllRules();
-
+        List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
-        return getFileAttributes(kieSession);
+        kieSession.dispose();
+        return resultingFileAttributes;
     }
 
 
@@ -127,7 +130,7 @@ public class DroolsExecutionService {
         List<FileAttribute> fileAttributes = new LinkedList<>();
         QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes");
         for (QueryResultsRow resultsRow : entitiesResult) {
-            fileAttributes.add((FileAttribute) resultsRow.get("$result"));
+            fileAttributes.add((FileAttribute) resultsRow.get("$fileAttribute"));
         }
         return fileAttributes;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -11,9 +11,9 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
@@ -75,7 +75,11 @@ public class RedactionLogCreatorService {
     private RedactionLogEntry createRedactionLogEntry(RedactionEntity entity, String dossierTemplateId) {
 
         Set<String> referenceIds = new HashSet<>();
-        entity.getReferences().forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        entity.getReferences()
+                .stream()
+                .filter(redactionEntity -> !redactionEntity.isRemoved())
+                .forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        int sectionNumber = entity.getDeepestFullyContainingNode().getTocId().isEmpty() ? -1 : entity.getDeepestFullyContainingNode().getTocId().get(0);
 
         return RedactionLogEntry.builder()
                 .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
@@ -88,7 +92,7 @@ public class RedactionLogCreatorService {
                 .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
                 .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
                 .section(entity.getDeepestFullyContainingNode().toString())
-                .sectionNumber(entity.getDeepestFullyContainingNode().getTocId().get(0))
+                .sectionNumber(sectionNumber)
                 .matchedRule(entity.getMatchedRule())
                 .isDictionaryEntry(entity.isDictionaryEntry())
                 .textAfter(entity.getTextAfter().toString())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -15,9 +15,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SectionText;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
 
 import lombok.RequiredArgsConstructor;
 
@@ -52,9 +52,9 @@ public class SectionGridCreatorService {
                     continue;
                 }
 
-                if (textBlock instanceof TextBlock) {
+                if (textBlock instanceof ClassificationTextBlock) {
 
-                    TextBlock tb = (TextBlock) textBlock;
+                    ClassificationTextBlock tb = (ClassificationTextBlock) textBlock;
                     classifiedDoc.getSectionGrid()
                             .getRectanglesPerPage()
                             .computeIfAbsent(page, (x) -> new ArrayList<>())
@@ -65,10 +65,10 @@ public class SectionGridCreatorService {
                                     paragraph.getPageBlocks().size(),
                                     null));
 
-                } else if (textBlock instanceof Table) {
+                } else if (textBlock instanceof TablePageBlock) {
 
                     List<CellRectangle> cellRectangles = new ArrayList<>();
-                    for (List<Cell> row : ((Table) textBlock).getRows()) {
+                    for (List<Cell> row : ((TablePageBlock) textBlock).getRows()) {
                         for (Cell cell : row) {
                             if (cell != null) {
                                 cellRectangles.add(new CellRectangle(new Point((float) cell.getX(), (float) cell.getY()), (float) cell.getWidth(), (float) cell.getHeight()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -19,9 +19,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SectionText;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
@@ -42,8 +42,8 @@ public class SectionTextBuilderService {
         AtomicInteger sectionNumber = new AtomicInteger(1);
         for (Section section : classifiedDoc.getSections()) {
 
-            List<Table> tables = section.getTables();
+            List<TablePageBlock> tables = section.getTables();
-            for (Table table : tables) {
+            for (TablePageBlock table : tables) {
                 sectionTexts.addAll(processTablePerRow(table, sectionNumber));
                 sectionNumber.incrementAndGet();
             }
@@ -70,14 +70,14 @@ public class SectionTextBuilderService {
     }
 
 
-    private List<SectionText> processTablePerRow(Table table, AtomicInteger sectionNumber) {
+    private List<SectionText> processTablePerRow(TablePageBlock table, AtomicInteger sectionNumber) {
 
         List<SectionText> sectionTexts = new ArrayList<>();
         boolean hasHeader = hasTableHeader(table);
 
         for (List<Cell> row : table.getRows()) {
 
-            List<TextBlock> textBlocks = new ArrayList<>();
+            List<ClassificationTextBlock> textBlocks = new ArrayList<>();
             List<SectionArea> areas = new ArrayList<>();
             Map<String, CellValue> tabularData = new HashMap<>();
             List<Integer> startOffsets = new ArrayList<>();
@@ -125,13 +125,13 @@ public class SectionTextBuilderService {
     }
 
 
-    public String getRowText(List<TextBlock> rowTextBlocks) {
+    public String getRowText(List<ClassificationTextBlock> rowTextBlocks) {
 
         return SearchableText.buildString(rowTextBlocks.stream().map(textBlock -> textBlock.getSequences()).flatMap(List::stream).collect(Collectors.toList()));
     }
 
 
-    private boolean hasTableHeader(Table table) {
+    private boolean hasTableHeader(TablePageBlock table) {
 
         return table.getRows().stream().anyMatch(row -> row.stream().anyMatch(cell -> !cell.isHeaderCell() && !cell.getHeaderCells().isEmpty()));
     }
@@ -170,13 +170,13 @@ public class SectionTextBuilderService {
 
 
     private SectionText processText(SearchableText searchableText,
-                                    List<TextBlock> paragraphTextBlocks,
+                                    List<ClassificationTextBlock> paragraphTextBlocks,
                                     String headline,
                                     AtomicInteger sectionNumber,
                                     List<ClassifiedImage> images) {
 
         SectionText sectionText = new SectionText();
-        for (TextBlock paragraphTextBlock : paragraphTextBlocks) {
+        for (ClassificationTextBlock paragraphTextBlock : paragraphTextBlocks) {
             SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()),
                     paragraphTextBlock.getWidth(),
                     paragraphTextBlock.getHeight(),

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -36,7 +36,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.se
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
@@ -83,7 +83,6 @@ public class AnalyzeService {
     ImageServiceResponseAdapter imageServiceResponseAdapter;
     ImportedRedactionService importedRedactionService;
     SectionFinder sectionFinder;
-    DocumentGraphFactory documentGraphFactory;
 
     FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
@@ -112,7 +111,7 @@ public class AnalyzeService {
             throw new RedactionException(e);
         }
 
-        DocumentGraph documentGraph = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+        DocumentGraph documentGraph = DocumentGraphFactory.buildDocumentGraph(classifiedDoc);
 
         List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
 
@@ -206,8 +205,10 @@ public class AnalyzeService {
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
         long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-
+        log.debug("Starting Dictionary Search");
+        long dictSearchStart = System.currentTimeMillis();
         entityRedactionService.addDictionaryEntities(dictionary, documentGraph);
+        log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
         Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntitiesToGraphAndReturnAddedFileAttributes(dictionary,
                 documentGraph,
                 kieContainer,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java

@@ -19,7 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
 
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -10,12 +11,14 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
 
 import lombok.AccessLevel;
@@ -39,12 +42,15 @@ public class EntityRedactionService {
                                                                                  AnalyzeRequest analyzeRequest,
                                                                                  List<EntityRecognitionEntity> nerEntities) {
 
+        log.debug("Starting Drools Execution");
+        long droolsStart = System.currentTimeMillis();
         List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer,
                 documentGraph,
                 dictionary,
                 analyzeRequest.getFileAttributes(),
                 analyzeRequest.getManualRedactions(),
                 nerEntities);
+        log.debug("Finished Drools Execution in {} ms", System.currentTimeMillis() - droolsStart);
         return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
     }
 
@@ -67,17 +73,30 @@ public class EntityRedactionService {
     }
 
 
-    public void addDictionaryEntities(Dictionary dictionary, DocumentGraph documentGraph) {
+    public void addDictionaryEntities(Dictionary dictionary, DocumentGraph document) {
 
-        dictionary.getDictionaryModels().forEach(dictionaryModel -> addDictionaryModelEntities(dictionaryModel, documentGraph));
+        List<RedactionEntity> foundEntities = new LinkedList<>();
+        for (var model : dictionary.getDictionaryModels()) {
+            findEntitiesWithSearchImplementation(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
+            findEntitiesWithSearchImplementation(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
+            findEntitiesWithSearchImplementation(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
+        }
+        foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document));
     }
 
 
-    private void addDictionaryModelEntities(DictionaryModel dictionaryModel, DocumentGraph documentGraph) {
+    private void findEntitiesWithSearchImplementation(DocumentGraph documentGraph,
+                                                      SearchImplementation searchImplementation,
+                                                      EntityType entityType,
+                                                      List<RedactionEntity> foundEntities,
+                                                      String type) {
 
-        entityCreationService.bySearchImplementation(dictionaryModel.getEntriesSearch(), dictionaryModel.getType(), EntityType.ENTITY, documentGraph);
+        TextBlock textBlock = documentGraph.getTextBlock();
-        entityCreationService.bySearchImplementation(dictionaryModel.getFalsePositiveSearch(), dictionaryModel.getType(), EntityType.FALSE_POSITIVE, documentGraph);
+        searchImplementation.getBoundaries(textBlock, textBlock.getBoundary())
-        entityCreationService.bySearchImplementation(dictionaryModel.getFalseRecommendationsSearch(), dictionaryModel.getType(), EntityType.FALSE_RECOMMENDATION, documentGraph);
+                .stream()
+                .filter(boundary -> entityCreationService.isValidEntityBoundary(textBlock, boundary))
+                .map(bounds -> RedactionEntity.initialEntityNode(bounds, type, entityType))
+                .forEach(foundEntities::add);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Liszt.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.util.List;
+
+/**
+ * Drools fucked up and introduced a bug with newer versions of java, this is needed or else:
+ * java.lang.IncompatibleClassChangeError: Method 'java.util.List java.util.List.of(java.lang.Object, java.lang.Object)' must be InterfaceMethodref constant
+ * Will be thrown on LHS of rules
+ */
+public class Liszt {
+
+    public static <E> List<E> of(E e1, E e2) {
+
+        return List.of(e1, e2);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java

@@ -12,8 +12,8 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -15,8 +15,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -46,12 +46,12 @@ public class PdfVisualisationService {
                     if (textBlock.getPage() != page) {
                         continue;
                     }
-                    if (textBlock instanceof TextBlock) {
+                    if (textBlock instanceof ClassificationTextBlock) {
                         textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
-                        visualizeTextBlock((TextBlock) textBlock, contentStream);
+                        visualizeTextBlock((ClassificationTextBlock) textBlock, contentStream);
-                    } else if (textBlock instanceof Table) {
+                    } else if (textBlock instanceof TablePageBlock) {
                         textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
-                        visualizeTable((Table) textBlock, contentStream);
+                        visualizeTable((TablePageBlock) textBlock, contentStream);
                     }
 
                 }
@@ -75,10 +75,10 @@ public class PdfVisualisationService {
                 if (textBlock == null) {
                     continue;
                 }
-                if (textBlock instanceof TextBlock) {
+                if (textBlock instanceof ClassificationTextBlock) {
-                    visualizeTextBlock((TextBlock) textBlock, contentStream);
+                    visualizeTextBlock((ClassificationTextBlock) textBlock, contentStream);
-                } else if (textBlock instanceof Table) {
+                } else if (textBlock instanceof TablePageBlock) {
-                    visualizeTable((Table) textBlock, contentStream);
+                    visualizeTable((TablePageBlock) textBlock, contentStream);
                 }
             }
 
@@ -95,7 +95,7 @@ public class PdfVisualisationService {
     }
 
 
-    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
+    private void visualizeTextBlock(ClassificationTextBlock textBlock, PDPageContentStream contentStream) throws IOException {
 
         contentStream.setStrokingColor(Color.RED);
 
@@ -121,7 +121,7 @@ public class PdfVisualisationService {
 
 
     @SneakyThrows
-    private void drawPositions(PDPageContentStream contentStream, TextBlock textBlock) {
+    private void drawPositions(PDPageContentStream contentStream, ClassificationTextBlock textBlock) {
 
         contentStream.setNonStrokingColor(Color.BLUE);
         contentStream.setFont(PDType1Font.TIMES_ROMAN, 2f);
@@ -145,7 +145,7 @@ public class PdfVisualisationService {
     }
 
 
-    private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
+    private void visualizeTable(TablePageBlock table, PDPageContentStream contentStream) throws IOException {
 
         for (List<Cell> row : table.getRows()) {
             for (Cell cell : row) {
@@ -156,7 +156,7 @@ public class PdfVisualisationService {
                     contentStream.stroke();
 
                     contentStream.setStrokingColor(Color.GREEN);
-                    for (TextBlock textBlock : cell.getTextBlocks()) {
+                    for (ClassificationTextBlock textBlock : cell.getTextBlocks()) {
                         contentStream.addRect(textBlock.getPdfMinX(),
                                 textBlock.getPdfMinY(),
                                 textBlock.getPdfMaxX() - textBlock.getPdfMinX(),

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -162,7 +162,8 @@ public abstract class AbstractRedactionIntegrationTest {
         when(dictionaryClient.getDictionaryForType(REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false));
         when(dictionaryClient.getDictionaryForType(HINT_ONLY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY_INDICATOR, false));
         when(dictionaryClient.getDictionaryForType(MUST_REDACT_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT_INDICATOR, false));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR,
+                false));
         when(dictionaryClient.getDictionaryForType(TEST_METHOD_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD_INDICATOR, false));
         when(dictionaryClient.getDictionaryForType(PII_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_PII, false));
         when(dictionaryClient.getDictionaryForType(PURITY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY_INDICATOR, false));
@@ -172,8 +173,10 @@ public abstract class AbstractRedactionIntegrationTest {
         when(dictionaryClient.getDictionaryForType(SIGNATURE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE_INDICATOR, false));
         when(dictionaryClient.getDictionaryForType(FORMULA_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA_INDICATOR, false));
         when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE_INDICATOR, false));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR,true));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR,
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,true));
+                true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,
+                true));
 
     }
 
@@ -237,7 +240,7 @@ public abstract class AbstractRedactionIntegrationTest {
 
         URL resource = ResourceLoader.class.getClassLoader().getResource(path);
         if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+            throw new IllegalArgumentException("could not load classpath resource: drools/prod_rules_new.drl");
         }
         List<String> stringList = Files.readAllLines(new File(resource.getPath()).toPath());
         return String.join("\n", stringList);
@@ -426,22 +429,30 @@ public abstract class AbstractRedactionIntegrationTest {
     @SneakyThrows
     protected AnalyzeRequest uploadFileToStorage(String file) {
 
-        return prepareStorage(file, "files/cv_service_empty_response.json");
+        return prepareStorage(file, "files/cv_service_empty_response.json", "files/empty_image_response.json");
     }
 
 
     @SneakyThrows
     protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        return prepareStorage(file, cvServiceResponseFile, "files/empty_image_response.json");
-        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
-
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
     }
 
 
     @SneakyThrows
-    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
+    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile, String imageServiceResponseFile) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+        ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponseFile);
+
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageServiceResponseFileResource.getInputStream());
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageServiceResponseStream) {
 
         AnalyzeRequest request = AnalyzeRequest.builder()
                 .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
@@ -450,10 +461,13 @@ public abstract class AbstractRedactionIntegrationTest {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
         storageService.storeObject(TenantContext.getTenantId(),
                 RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
                 cvServiceResponseFileStream);
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
+                imageServiceResponseStream);
 
         return request;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java

@@ -1,546 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import static org.mockito.Mockito.when;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.kie.api.KieServices;
-import org.kie.api.builder.KieBuilder;
-import org.kie.api.builder.KieFileSystem;
-import org.kie.api.builder.KieModule;
-import org.kie.api.builder.KieRepository;
-import org.kie.api.builder.ReleaseId;
-import org.kie.api.runtime.KieContainer;
-import org.kie.internal.io.ResourceFactory;
-import org.mockito.stubbing.Answer;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
-@ExtendWith(SpringExtension.class)
-@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(AbstractTestWithDictionaries.TestConfiguration.class)
-public class AbstractTestWithDictionaries {
-
-    protected static final String RULES = loadFromClassPath("drools/testRules.drl");
-    protected static final String RULES_PATH = "drools/rules.drl";
-    protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
-    protected static final String MERGE_ENTITY_RULES_PATH = "drools/testRules.drl";
-    private static final String VERTEBRATE = "vertebrate";
-    private static final String ADDRESS = "CBI_address";
-    private static final String AUTHOR = "CBI_author";
-    private static final String SPONSOR = "CBI_sponsor";
-    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
-    private static final String REDACTION_INDICATOR = "redaction_indicator";
-    private static final String HINT_ONLY = "hint_only";
-    private static final String MUST_REDACT = "must_redact";
-    private static final String PUBLISHED_INFORMATION = "published_information";
-    private static final String TEST_METHOD = "test_method";
-    private static final String PURITY = "purity";
-    private static final String IMAGE = "image";
-    private static final String LOGO = "logo";
-    private static final String SIGNATURE = "signature";
-    private static final String FORMULA = "formula";
-    private static final String OCR = "ocr";
-    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
-    private static final String IMPORTED_REDACTION = "imported_redaction";
-    private static final String PII = "PII";
-    private static final String ROTATE_SIMPLE = "RotateSimple";
-
-    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
-    protected final static String TEST_DOSSIER_ID = "123";
-    protected final static String TEST_FILE_ID = "123";
-
-    @Autowired
-    protected StorageService storageService;
-
-    @MockBean
-    protected DictionaryClient dictionaryClient;
-
-    @MockBean
-    protected RulesClient rulesClient;
-
-    private final Map<String, List<String>> dictionary = new HashMap<>();
-    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
-    private final Map<String, List<String>> falsePositive = new HashMap<>();
-    private final Map<String, List<String>> falseRecommendation = new HashMap<>();
-    private final Map<String, String> typeColorMap = new HashMap<>();
-    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
-    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
-    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
-    private final Map<String, Integer> rankTypeMap = new HashMap<>();
-    private final Colors colors = new Colors();
-    private final Map<String, Long> reanalysisVersions = new HashMap<>();
-    private final Set<String> deleted = new HashSet<>();
-
-
-    @BeforeEach
-    public void stubClients() {
-
-        TenantContext.setTenantId("redaction");
-
-        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
-
-        loadDictionaryForTest();
-        loadTypeForTest();
-        loadNerForTest();
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
-
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS)
-                .dossierTemplateId(TEST_DOSSIER_ID)
-                .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
-                .build()));
-
-        mockDictionaryCalls(null);
-        mockDictionaryCalls(0L);
-
-        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
-    }
-
-
-    private void mockDictionaryCalls(Long version) {
-
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
-                false));
-        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
-        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                NO_REDACTION_INDICATOR,
-                false));
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                REDACTION_INDICATOR,
-                false));
-        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
-                false));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                PUBLISHED_INFORMATION,
-                false));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
-                false));
-        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
-        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
-        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
-        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
-        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
-        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
-        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
-        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
-                false));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                DOSSIER_REDACTIONS,
-                true));
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                IMPORTED_REDACTION,
-                true));
-
-    }
-
-
-    private static String loadFromClassPath(String path) {
-
-        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
-        }
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
-            StringBuilder sb = new StringBuilder();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
-        }
-    }
-
-
-    private String cleanDictionaryEntry(String entry) {
-
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-
-
-    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
-
-        if (entries == null) {
-            entries = Collections.emptyList();
-        }
-
-        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
-        entries.forEach(entry -> {
-            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
-        });
-        return dictionaryEntries;
-    }
-
-
-    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
-
-        return Type.builder()
-                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .hexColor(typeColorMap.get(type))
-                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
-                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
-                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
-                .isHint(hintTypeMap.get(type))
-                .isCaseInsensitive(caseInSensitiveMap.get(type))
-                .isRecommendation(recommendationTypeMap.get(type))
-                .rank(rankTypeMap.get(type))
-                .build();
-    }
-
-
-    private void loadTypeForTest() {
-
-        typeColorMap.put(VERTEBRATE, "#ff85f7");
-        typeColorMap.put(ADDRESS, "#ffe187");
-        typeColorMap.put(AUTHOR, "#ffe187");
-        typeColorMap.put(SPONSOR, "#85ebff");
-        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
-        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
-        typeColorMap.put(HINT_ONLY, "#abc0c4");
-        typeColorMap.put(MUST_REDACT, "#fab4c0");
-        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
-        typeColorMap.put(TEST_METHOD, "#91fae8");
-        typeColorMap.put(PII, "#66ccff");
-        typeColorMap.put(PURITY, "#ffe187");
-        typeColorMap.put(IMAGE, "#fcc5fb");
-        typeColorMap.put(OCR, "#fcc5fb");
-        typeColorMap.put(LOGO, "#ffe187");
-        typeColorMap.put(FORMULA, "#ffe187");
-        typeColorMap.put(SIGNATURE, "#ffe187");
-        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
-        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
-
-        hintTypeMap.put(VERTEBRATE, true);
-        hintTypeMap.put(ADDRESS, false);
-        hintTypeMap.put(AUTHOR, false);
-        hintTypeMap.put(SPONSOR, false);
-        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
-        hintTypeMap.put(REDACTION_INDICATOR, true);
-        hintTypeMap.put(HINT_ONLY, true);
-        hintTypeMap.put(MUST_REDACT, true);
-        hintTypeMap.put(PUBLISHED_INFORMATION, true);
-        hintTypeMap.put(TEST_METHOD, true);
-        hintTypeMap.put(PII, false);
-        hintTypeMap.put(PURITY, false);
-        hintTypeMap.put(IMAGE, true);
-        hintTypeMap.put(OCR, true);
-        hintTypeMap.put(FORMULA, false);
-        hintTypeMap.put(LOGO, false);
-        hintTypeMap.put(SIGNATURE, false);
-        hintTypeMap.put(DOSSIER_REDACTIONS, false);
-        hintTypeMap.put(IMPORTED_REDACTION, false);
-        hintTypeMap.put(ROTATE_SIMPLE, false);
-
-        caseInSensitiveMap.put(VERTEBRATE, true);
-        caseInSensitiveMap.put(ADDRESS, false);
-        caseInSensitiveMap.put(AUTHOR, false);
-        caseInSensitiveMap.put(SPONSOR, false);
-        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(HINT_ONLY, true);
-        caseInSensitiveMap.put(MUST_REDACT, true);
-        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
-        caseInSensitiveMap.put(TEST_METHOD, false);
-        caseInSensitiveMap.put(PII, false);
-        caseInSensitiveMap.put(PURITY, false);
-        caseInSensitiveMap.put(IMAGE, true);
-        caseInSensitiveMap.put(OCR, true);
-        caseInSensitiveMap.put(SIGNATURE, true);
-        caseInSensitiveMap.put(LOGO, true);
-        caseInSensitiveMap.put(FORMULA, true);
-        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
-        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
-        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
-
-        recommendationTypeMap.put(VERTEBRATE, false);
-        recommendationTypeMap.put(ADDRESS, false);
-        recommendationTypeMap.put(AUTHOR, false);
-        recommendationTypeMap.put(SPONSOR, false);
-        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(HINT_ONLY, false);
-        recommendationTypeMap.put(MUST_REDACT, false);
-        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
-        recommendationTypeMap.put(TEST_METHOD, false);
-        recommendationTypeMap.put(PII, false);
-        recommendationTypeMap.put(PURITY, false);
-        recommendationTypeMap.put(IMAGE, false);
-        recommendationTypeMap.put(OCR, false);
-        recommendationTypeMap.put(FORMULA, false);
-        recommendationTypeMap.put(SIGNATURE, false);
-        recommendationTypeMap.put(LOGO, false);
-        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
-        recommendationTypeMap.put(IMPORTED_REDACTION, false);
-        recommendationTypeMap.put(ROTATE_SIMPLE, false);
-
-        rankTypeMap.put(PURITY, 155);
-        rankTypeMap.put(PII, 150);
-        rankTypeMap.put(ADDRESS, 140);
-        rankTypeMap.put(AUTHOR, 130);
-        rankTypeMap.put(SPONSOR, 120);
-        rankTypeMap.put(VERTEBRATE, 110);
-        rankTypeMap.put(MUST_REDACT, 100);
-        rankTypeMap.put(REDACTION_INDICATOR, 90);
-        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
-        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
-        rankTypeMap.put(TEST_METHOD, 60);
-        rankTypeMap.put(HINT_ONLY, 50);
-        rankTypeMap.put(IMAGE, 30);
-        rankTypeMap.put(OCR, 29);
-        rankTypeMap.put(LOGO, 28);
-        rankTypeMap.put(SIGNATURE, 27);
-        rankTypeMap.put(FORMULA, 26);
-        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
-        rankTypeMap.put(IMPORTED_REDACTION, 200);
-        rankTypeMap.put(ROTATE_SIMPLE, 150);
-
-        colors.setSkippedColor("#cccccc");
-        colors.setRequestAddColor("#04b093");
-        colors.setRequestRemoveColor("#04b093");
-    }
-
-
-    @SneakyThrows
-    private void loadNerForTest() {
-
-        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
-                responseJson.getInputStream());
-    }
-
-
-    private List<Type> getTypeResponse() {
-
-        return typeColorMap.entrySet()
-                .stream()
-                .map(typeColor -> Type.builder()
-                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                        .type(typeColor.getKey())
-                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                        .hexColor(typeColor.getValue())
-                        .isHint(hintTypeMap.get(typeColor.getKey()))
-                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
-                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
-                        .rank(rankTypeMap.get(typeColor.getKey()))
-                        .build())
-
-                .collect(Collectors.toList());
-    }
-
-
-    protected List<File> getPathsRecursively(File path) {
-
-        List<File> result = new ArrayList<>();
-        if (path == null || path.listFiles() == null) {
-            return result;
-        }
-        for (File f : path.listFiles()) {
-            if (f.isFile()) {
-                result.add(f);
-            } else {
-                result.addAll(getPathsRecursively(f));
-            }
-        }
-        return result;
-
-    }
-
-
-    protected void loadOnlyDictionaryForSimpleFile() {
-
-        dictionary.clear();
-        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-    }
-
-
-    private void loadDictionaryForTest() {
-
-        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
-
-        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(String file) {
-
-        return prepareStorage(file, "files/cv_service_empty_response.json", "files/empty_image_response.json");
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile) {
-
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
-        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
-        ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
-
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream());
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream) {
-
-        AnalyzeRequest request = AnalyzeRequest.builder()
-                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .lastProcessed(OffsetDateTime.now())
-                .build();
-
-        storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
-                imageInfoStream);
-        storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
-                cvServiceResponseFileStream);
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
-
-        return request;
-    }
-
-
-    @AfterEach
-    public void cleanupStorage() {
-
-        if (this.storageService instanceof FileSystemBackedStorageService) {
-            ((FileSystemBackedStorageService) this.storageService).clearStorage();
-        }
-    }
-
-
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
-    public static class TestConfiguration {
-
-        @Bean
-        public KieContainer kieContainer() {
-
-            KieServices kieServices = KieServices.Factory.get();
-
-            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            kieFileSystem.write(ResourceFactory.newClassPathResource(RULES_PATH, "UTF-8"));
-
-            KieRepository kieRepository = kieServices.getRepository();
-
-            kieRepository.addKieModule(new KieModule() {
-                public ReleaseId getReleaseId() {
-
-                    return kieRepository.getDefaultReleaseId();
-                }
-            });
-
-            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem).buildAll();
-
-            return kieServices.newKieContainer(kieRepository.getDefaultReleaseId());
-        }
-
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService();
-        }
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphPerformanceIntegrationTest.java

@@ -2,50 +2,59 @@ package com.iqser.red.service.redaction.v1.server;
 
 import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 import static com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw.drawRectangle2DList;
+import static org.mockito.Mockito.when;
 
 import java.awt.Color;
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
-import org.kie.api.runtime.KieSession;
+import org.kie.internal.io.ResourceFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentGraphIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
 
 import lombok.SneakyThrows;
 
-public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
+@Import(DocumentGraphPerformanceIntegrationTest.TestConfiguration.class)
+public class DocumentGraphPerformanceIntegrationTest extends BuildDocumentGraphIntegrationTest {
 
-    @Autowired
+    private static final String RULES = "drools/rules.drl";
-    private DocumentGraphFactory documentGraphFactory;
-
-    @Autowired
-    private PdfSegmentationService segmentationService;
 
     @Autowired
     private DictionaryService dictionaryService;
@@ -54,24 +63,71 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
     private EntityCreationService entityCreationService;
 
     @Autowired
-    private RedactionLogCreatorService redactionLogCreatorService;
+    private DroolsExecutionService droolsExecutionService;
 
     @Qualifier("kieContainer")
     @Autowired
     private KieContainer kieContainer;
 
+    @Configuration
+    @Import(BuildDocumentGraphIntegrationTest.TestConfiguration.class)
+    public static class TestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            kieFileSystem.write(ResourceFactory.newClassPathResource(RULES));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+    }
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        TenantContext.setTenantId("redaction");
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
+
+        mockDictionaryCalls(null);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
 
     @Test
     @SneakyThrows
     public void testDroolsOnDocumentGraph() {
 
         String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
-
+        DocumentGraph document = buildGraph(filename);
-        prepareStorage(filename + ".pdf");
-        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
-
-        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
-        DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
 
         dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
@@ -88,28 +144,20 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
         long graphInsertionStart = System.currentTimeMillis();
         foundEntities.forEach(entity -> entity.setDossierDictionaryEntry(true));
         foundEntities.forEach(entity -> entity.setDictionaryEntry(true));
-        foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document.getTableOfContents()));
+        foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, document));
         System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart);
 
-        KieSession kieSession = kieContainer.newKieSession();
+        long droolsStart = System.currentTimeMillis();
-        kieSession.setGlobal("document", document);
+        List<FileAttribute> fileAttributes = droolsExecutionService.executeRules(kieContainer,
-        kieSession.setGlobal("entityCreationService", entityCreationService);
+                document,
-        kieSession.setGlobal("dictionary", dictionary);
+                dictionary,
+                Collections.emptyList(),
+                new ManualRedactions(),
+                Collections.emptyList());
+        System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart);
 
-        long serializationStart = System.currentTimeMillis();
-        document.getEntities().forEach(kieSession::insert);
-        document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode()));
-        document.getPages().forEach(kieSession::insert);
-        System.out.printf("Object serialization and kieSession insertion took %d ms\n", System.currentTimeMillis() - serializationStart);
-
-        long dictionaryAddsStart = System.currentTimeMillis();
-        kieSession.insert(FileAttribute.builder().label("Vertebrate Study").value("Yes").build());
-        kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
-        kieSession.fireAllRules();
-        System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - dictionaryAddsStart);
         System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart);
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID);
+        drawAllEntities(filename, document);
-        drawAllEntities(filename, fileResource, document);
     }
 
 
@@ -119,12 +167,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
 
         int n = 10000;
         String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
-
+        DocumentGraph document = buildGraph(filename);
-        prepareStorage(filename + ".pdf");
-        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
-
-        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
-        DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
         var start = System.currentTimeMillis();
         for (int i = 0; i < n; i++) {
             TextBlock textBlock = document.getTextBlock();
@@ -155,17 +198,12 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
     public void testDictionarySearchOnDocumentGraph() {
 
         String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
-
+        DocumentGraph document = buildGraph(filename);
-        prepareStorage(filename + ".pdf");
-        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
-
-        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
-        DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
 
         dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
 
-        int numberOfRuns = 25;
+        int numberOfRuns = 1;
         float totalSearchTime = 0;
         float totalGraphTime = 0;
         float totalInsertTime = 0;
@@ -174,7 +212,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
             foundEntities = new LinkedList<>();
 
             var graphStart = System.currentTimeMillis();
-            document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+            document = buildGraph(filename);
             var graphTime = System.currentTimeMillis() - graphStart;
             totalGraphTime += graphTime;
 
@@ -189,7 +227,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
 
             var insertStart = System.currentTimeMillis();
             DocumentGraph finalDocument = document;
-            foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument.getTableOfContents()));
+            foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument));
             var insertTime = System.currentTimeMillis() - insertStart;
             totalInsertTime += insertTime;
 
@@ -202,13 +240,14 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
         System.out.printf("Found %d entities and saved %d\n", foundEntities.size(), document.getEntities().size());
         assert document.getEntities().size() == foundEntities.size();
 
-        drawAllEntities(filename, fileResource, document);
+        drawAllEntities(filename, document);
     }
 
 
-    private static void drawAllEntities(String filename, ClassPathResource fileResource, DocumentGraph document) throws IOException {
+    private static void drawAllEntities(String filename, DocumentGraph document) throws IOException {
 
         var tmpFileName = "/tmp/" + filename.split("/")[2] + "_ENTITY_BBOX.pdf";
+        var fileResource = new ClassPathResource(filename + ".pdf");
         try (var fileStream = fileResource.getInputStream()) {
             PDDocument pdDocument = PDDocument.load(fileStream);
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -44,7 +44,7 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.*;
+
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
@@ -53,6 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@@ -202,7 +204,7 @@ public class HeadlinesGoldStandardIntegrationTest {
 
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class/*, StorageAutoConfiguration.class*/})
-    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
     public static class RedactionIntegrationTestConfiguration {
 
         @Bean
@@ -365,7 +367,9 @@ public class HeadlinesGoldStandardIntegrationTest {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
+                cvServiceResponseFileStream);
         storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
 
         return request;
@@ -376,7 +380,7 @@ public class HeadlinesGoldStandardIntegrationTest {
 
         URL resource = ResourceLoader.class.getClassLoader().getResource(path);
         if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+            throw new IllegalArgumentException("could not load classpath resource: drools/prod_rules_new.drl");
         }
         try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
             StringBuilder sb = new StringBuilder();
@@ -395,7 +399,9 @@ public class HeadlinesGoldStandardIntegrationTest {
     private void loadNerForTest() {
 
         ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                responseJson.getInputStream());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -227,15 +227,16 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
 
+        System.out.println("Start Full integration test");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
 
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
-
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -95,7 +95,7 @@ import lombok.extern.slf4j.Slf4j;
 @Import(RulesTest.RulesTestConfiguration.class)
 public class RulesTest {
 
-    private static final String RULES_PATH = "drools/rules.drl";
+    private static final String RULES_PATH = "drools/prod_rules_new.drl";
     private static final String RULES = loadFromClassPath(RULES_PATH);
     private static final String VERTEBRATE = "vertebrate";
     private static final String ADDRESS = "CBI_address";
@@ -797,7 +797,7 @@ public class RulesTest {
 
         URL resource = ResourceLoader.class.getClassLoader().getResource(path);
         if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+            throw new IllegalArgumentException("could not load classpath resource: drools/prod_rules_new.drl");
         }
         try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
             StringBuilder sb = new StringBuilder();
@@ -814,7 +814,7 @@ public class RulesTest {
 
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
     public static class RulesTestConfiguration {
 
         @Bean

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphIntegrationTest.java

@@ -0,0 +1,78 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.ImageServiceResponseAdapter;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(BuildDocumentGraphIntegrationTest.TestConfiguration.class)
+public class BuildDocumentGraphIntegrationTest extends AbstractRedactionIntegrationTest {
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    public static class TestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+    @Autowired
+    private PdfSegmentationService segmentationService;
+
+    @Autowired
+    private ImageServiceResponseAdapter imageServiceResponseAdapter;
+
+
+    @SneakyThrows
+    protected DocumentGraph buildGraph(String filename) {
+
+        if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
+            prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json");
+        } else {
+            uploadFileToStorage(filename + ".pdf");
+        }
+        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+
+        try (InputStream inputStream = fileResource.getInputStream()) {
+            Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID);
+            Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages);
+            return DocumentGraphFactory.buildDocumentGraph(classifiedDoc);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphTest.java

@@ -1,62 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.graph;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
-import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.ImageServiceResponseAdapter;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
-
-import lombok.SneakyThrows;
-
-public class BuildDocumentGraphTest extends AbstractTestWithDictionaries {
-
-    @Autowired
-    private DocumentGraphFactory documentGraphFactory;
-
-    @Autowired
-    private PdfSegmentationService segmentationService;
-
-    @Autowired
-    private ImageServiceResponseAdapter imageServiceResponseAdapter;
-
-
-    @Test
-    public void buildMetolachlor() {
-
-        DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
-        assertEquals(221, documentGraph.getPages().size());
-        assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
-        assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
-    }
-
-
-    @SneakyThrows
-    protected DocumentGraph buildGraph(String filename) {
-
-        if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
-            prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json");
-        } else {
-            prepareStorage(filename + ".pdf");
-        }
-        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
-
-        try (InputStream inputStream = fileResource.getInputStream()) {
-            Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID);
-            Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages);
-            return documentGraphFactory.buildDocumentGraph(classifiedDoc);
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionIntegrationTest.java

@@ -10,9 +10,9 @@ import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.TableOfContents;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.HeadlineNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
@@ -24,7 +24,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.te
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 
-public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
+public class DocumentGraphEntityInsertionIntegrationTest extends BuildDocumentGraphIntegrationTest {
 
     @Autowired
     private EntityCreationService entityCreationService;
@@ -51,7 +51,7 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
         RedactionEntity redactionEntity = RedactionEntity.initialEntityNode(boundary, "123", EntityType.ENTITY);
-        entityCreationService.addEntityToGraph(redactionEntity, documentGraph.getTableOfContents());
+        entityCreationService.addEntityToGraph(redactionEntity, documentGraph);
         return redactionEntity;
     }
 
@@ -68,7 +68,7 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
         assertEquals(searchTerm, redactionEntity.getValue());
         assertEquals("Rule 5: Do not redact genitive CBI_authors (Entries based on Dict) ",
                 redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
-        assertEquals(2, redactionEntity.getIntersectingNodes().size());
+        assertEquals(3, redactionEntity.getIntersectingNodes().size());
         assertEquals(5, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
         assertInstanceOf(ParagraphNode.class, redactionEntity.getDeepestFullyContainingNode());
 
@@ -87,7 +87,7 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
         assertEquals(" Purity Hint", redactionEntity.getTextAfter());
         assertEquals(searchTerm, redactionEntity.getValue());
         assertEquals("Rule 39: Purity Hint ", redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
-        assertEquals(2, redactionEntity.getIntersectingNodes().size());
+        assertEquals(3, redactionEntity.getIntersectingNodes().size());
         assertEquals(6, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
         assertInstanceOf(HeadlineNode.class, redactionEntity.getDeepestFullyContainingNode());
 
@@ -106,7 +106,7 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
         assertEquals("", redactionEntity.getTextAfter());
         assertEquals(searchTerm, redactionEntity.getValue());
         assertEquals("Rule 6-11 (Authors Table) ", redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
-        assertEquals(3, redactionEntity.getIntersectingNodes().size());
+        assertEquals(5, redactionEntity.getIntersectingNodes().size());
         assertEquals(15, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
         assertInstanceOf(TableCellNode.class, redactionEntity.getDeepestFullyContainingNode());
 
@@ -189,7 +189,7 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
         assertEquals("1.1.4 Evaluations carried out under other regulatory contexts ",
                 redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
         assertEquals(searchTerm, redactionEntity.getValue());
-        assertEquals(2, redactionEntity.getIntersectingNodes().size());
+        assertEquals(3, redactionEntity.getIntersectingNodes().size());
         assertEquals(5, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
         assertTrue(redactionEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 10));
         assertInstanceOf(ParagraphNode.class, redactionEntity.getDeepestFullyContainingNode());
@@ -210,14 +210,14 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
         RedactionEntity redactionEntity = RedactionEntity.initialEntityNode(boundary, "123", EntityType.ENTITY);
-        entityCreationService.addEntityToGraph(redactionEntity, documentGraph.getTableOfContents());
+        entityCreationService.addEntityToGraph(redactionEntity, documentGraph);
 
         assertEquals("2.6.1 Summary of ", redactionEntity.getTextBefore());
         assertEquals(" and excretion in", redactionEntity.getTextAfter());
         assertEquals("2.6.1 Summary of absorption, distribution, metabolism and excretion in mammals ",
                 redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
         assertEquals(searchTerm, redactionEntity.getValue());
-        assertEquals(2, redactionEntity.getIntersectingNodes().size());
+        assertEquals(3, redactionEntity.getIntersectingNodes().size());
         assertEquals(4, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
         assertTrue(redactionEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 33));
         assertInstanceOf(HeadlineNode.class, redactionEntity.getDeepestFullyContainingNode());
@@ -236,8 +236,8 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
         assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", redactionEntity.getTextBefore());
         assertEquals(" of metabolite of", redactionEntity.getTextAfter());
         assertEquals(searchTerm, redactionEntity.getValue());
-        assertEquals(3, redactionEntity.getIntersectingNodes().size());
+        assertEquals(4, redactionEntity.getIntersectingNodes().size());
-        assertEquals("2.7.2 Summary of metabolism, distribution and expression of residues in plants, poultry, lactating ruminants, pigs and fish ",
+        assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ",
                 redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
         assertTrue(redactionEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 54));
         assertEquals(26, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
@@ -248,11 +248,12 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
     }
 
 
+    // this might fail, if an entity with the same name exists twice in the deepest containing node
     private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, RedactionEntity redactionEntity) {
 
         List<Integer> paragraphStart = redactionEntity.getIntersectingNodes().stream()//
                 .map(SemanticNode::buildTextBlock)//
-                .map(textBlock -> textBlock.indexOf(searchTerm))//
+                .map(textBlock -> textBlock.indexOf(searchTerm, redactionEntity.getDeepestFullyContainingNode().getBoundary().start()))//
                 .toList();
 
         paragraphStart.forEach(nodeStart -> assertEquals(redactionEntity.getBoundary().start(), nodeStart));
@@ -267,7 +268,7 @@ public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
 
         Boundary boundary = new Boundary(start, start + searchTerm.length());
         RedactionEntity redactionEntity = RedactionEntity.initialEntityNode(boundary, "123", EntityType.ENTITY);
-        entityCreationService.addEntityToGraph(redactionEntity, documentGraph.getTableOfContents());
+        entityCreationService.addEntityToGraph(redactionEntity, documentGraph);
         PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
 
         assertEquals(redactionEntity.getValue(), searchTerm);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingIntegrationTest.java

@@ -1,5 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.document.graph;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.wildfly.common.Assert.assertTrue;
+
 import org.junit.jupiter.api.Test;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData;
@@ -9,12 +12,15 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.Pag
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.TableOfContentsData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
 import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 
 import lombok.SneakyThrows;
 
-public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
+public class DocumentGraphMappingIntegrationTest extends BuildDocumentGraphIntegrationTest {
 
     @Test
     @SneakyThrows
@@ -45,8 +51,37 @@ public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
                 .build();
         DocumentGraph newDocumentGraph = DocumentGraphMapper.toDocumentGraph(documentData2);
 
-        assert document.toString().equals(newDocumentGraph.toString());
+        assertTrue(allTablesHavePositiveNumberOfRowsAndColumns(document));
-        assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString());
+        assertTrue(allTablesHavePositiveNumberOfRowsAndColumns(documentData));
+        assertTrue(allTablesHavePositiveNumberOfRowsAndColumns(documentData2));
+        assertTrue(allTablesHavePositiveNumberOfRowsAndColumns(newDocumentGraph));
+
+        assertEquals(document.toString(), newDocumentGraph.toString());
+        assertEquals(document.getTableOfContents().toString(), newDocumentGraph.getTableOfContents().toString());
+    }
+
+
+    private static boolean allTablesHavePositiveNumberOfRowsAndColumns(DocumentGraph documentGraph) {
+
+        return documentGraph.streamAllSubNodes()
+                .filter(semanticNode -> semanticNode instanceof TableNode)
+                .map(semanticNode -> (TableNode) semanticNode)
+                .allMatch(tableNode -> tableNode.getNumberOfCols() > 0 && tableNode.getNumberOfRows() > 0);
+    }
+
+
+    private static boolean allTablesHavePositiveNumberOfRowsAndColumns(DocumentData documentData) {
+
+        return documentData.getTableOfContents()
+                .streamAllEntries()
+                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
+                .map(TableOfContentsData.EntryData::getProperties)
+                .map(properties -> {
+                    var builder = TableNode.builder();
+                    PropertiesMapper.parseTableProperties(properties, builder);
+                    return builder.build();
+                })
+                .allMatch(tableNode -> tableNode.getNumberOfCols() > 0 && tableNode.getNumberOfRows() > 0);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTableNodeIntegrationTest.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableNode;
+
+public class DocumentGraphTableNodeIntegrationTest extends BuildDocumentGraphIntegrationTest {
+
+    @Test
+    public void testAllTableCellAccessesCorrect() {
+
+        DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06");
+        PageNode pageFive = documentGraph.getPages().stream().filter(pageNode -> pageNode.getNumber() == 5).findFirst().get();
+        TableNode tableNode = pageFive.getMainBody()
+                .stream()
+                .filter(semanticNode -> semanticNode instanceof TableNode)
+                .map(semanticNode -> (TableNode) semanticNode)
+                .findFirst()
+                .get();
+
+        for (int row = 0; row < tableNode.getNumberOfRows(); row++) {
+            for (int col = 0; col < tableNode.getNumberOfCols(); col++) {
+                TableCellNode tableCellNode = tableNode.getCell(row, col);
+                assertEquals(col, tableCellNode.getCol());
+                assertEquals(row, tableCellNode.getRow());
+            }
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphVisualizationIntegrationTest.java

@@ -9,13 +9,13 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
 
 import lombok.SneakyThrows;
 
-public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
+public class DocumentGraphVisualizationIntegrationTest extends BuildDocumentGraphIntegrationTest {
 
     @Test
     @SneakyThrows
@@ -31,6 +31,20 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    @Disabled
+    public void visualizeMetolachlor2() {
+
+        String filename = "files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06";
+
+        DocumentGraph documentGraph = buildGraph(filename);
+        TextBlock textBlock = documentGraph.buildTextBlock();
+
+        visualizeSemanticNodes(filename, documentGraph, textBlock);
+    }
+
+
     @Test
     @SneakyThrows
     @Disabled
@@ -45,6 +59,20 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    @Disabled
+    public void visualizeCraftedDocument() {
+
+        String filename = "files/new/crafted document";
+
+        DocumentGraph documentGraph = buildGraph(filename);
+        TextBlock textBlock = documentGraph.buildTextBlock();
+
+        visualizeSemanticNodes(filename, documentGraph, textBlock);
+    }
+
+
     private static void visualizeSemanticNodes(String filename, DocumentGraph documentGraph, TextBlock textBlock) throws IOException {
 
         var tmpFileName = "/tmp/" + filename.split("/")[2] + "_SEMANTIC_NODES_BBOX.pdf";

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java

@@ -0,0 +1,264 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.wildfly.common.Assert.assertFalse;
+
+import java.awt.geom.Rectangle2D;
+import java.time.OffsetDateTime;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
+import org.kie.api.runtime.KieSession;
+import org.kie.internal.io.ResourceFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ParagraphNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+
+@Import(ManualResizeRedactionIntegrationTest.TestConfiguration.class)
+public class ManualResizeRedactionIntegrationTest extends BuildDocumentGraphIntegrationTest {
+
+    private static final String RULES = "drools/manual_redaction_rules.drl";
+
+    @Autowired
+    private EntityCreationService entityCreationService;
+
+    @Autowired
+    private ManualRedactionApplicationService manualRedactionApplicationService;
+
+    @Qualifier("kieContainer")
+    @Autowired
+    private KieContainer kieContainer;
+
+    @Configuration
+    @Import(BuildDocumentGraphIntegrationTest.TestConfiguration.class)
+    public static class TestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            kieFileSystem.write(ResourceFactory.newClassPathResource(RULES));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+    }
+
+
+    @Test
+    public void manualResizeRedactionTest() {
+
+        DocumentGraph documentGraph = buildGraph("files/new/crafted document");
+        Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
+        Set<RedactionEntity> biggerEntities = entityCreationService.byKeyword("David Ksenia Max Mustermann", "CBI_author", EntityType.ENTITY, documentGraph);
+
+        RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
+        RedactionEntity biggerEntity = biggerEntities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
+
+        String initialId = entity.getRedactionPositionsPerPage().get(0).getId();
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId(initialId)
+                .value(biggerEntity.getValue())
+                .positions(toAnnotationRectangles(biggerEntity.getRedactionPositionsPerPage().get(0)))
+                .status(AnnotationStatus.APPROVED)
+                .build();
+
+        KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
+        kieSession.insert(entity);
+        kieSession.insert(manualResizeRedaction);
+        kieSession.fireAllRules();
+        kieSession.dispose();
+
+        assertEquals(biggerEntity.getBoundary(), entity.getBoundary());
+        assertEquals(biggerEntity.getDeepestFullyContainingNode(), entity.getDeepestFullyContainingNode());
+        assertEquals(biggerEntity.getIntersectingNodes(), entity.getIntersectingNodes());
+        assertEquals(biggerEntity.getPages(), entity.getPages());
+        assertEquals(biggerEntity.getValue(), entity.getValue());
+        assertEquals(initialId, entity.getRedactionPositionsPerPage().get(0).getId());
+        assertRectanglesAlmostEqual(biggerEntity.getRedactionPositionsPerPage().get(0).getRectanglePerLine(), entity.getRedactionPositionsPerPage().get(0).getRectanglePerLine());
+        assertTrue(entity.isResized());
+    }
+
+
+    @Test
+    public void manualForceRedactionTest() {
+
+        DocumentGraph documentGraph = buildGraph("files/new/crafted document");
+        Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
+
+        RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
+
+        String initialId = entity.getRedactionPositionsPerPage().get(0).getId();
+        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
+                .annotationId(initialId)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.now())
+                .build();
+
+        KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
+        kieSession.insert(entity);
+        kieSession.insert(manualForceRedaction);
+        kieSession.fireAllRules();
+        kieSession.dispose();
+
+        assertEquals(ParagraphNode.class, entity.getDeepestFullyContainingNode().getClass());
+        assertFalse(entity.getIntersectingNodes().isEmpty());
+        assertEquals(1, entity.getPages().size());
+        assertEquals("David Ksenia", entity.getValue());
+        assertEquals(initialId, entity.getRedactionPositionsPerPage().get(0).getId());
+        assertFalse(entity.isRemoved());
+        assertTrue(entity.isSkipRemoveEntitiesContainedInLarger());
+        assertTrue(entity.isRedaction());
+    }
+
+
+    @Test
+    public void manualIDRemovalTest() {
+
+        DocumentGraph documentGraph = buildGraph("files/new/crafted document");
+        Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
+
+        RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
+
+        String initialId = entity.getRedactionPositionsPerPage().get(0).getId();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build();
+
+        KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
+        kieSession.insert(entity);
+        kieSession.insert(idRemoval);
+        kieSession.fireAllRules();
+        kieSession.dispose();
+
+        assertNull(entity.getDeepestFullyContainingNode());
+        assertTrue(entity.getIntersectingNodes().isEmpty());
+        assertTrue(entity.getPages().isEmpty());
+        assertEquals("David Ksenia", entity.getValue());
+        assertEquals(initialId, entity.getRedactionPositionsPerPage().get(0).getId());
+        assertTrue(entity.isRemoved());
+    }
+
+
+    @Test
+    public void manualIDRemovalButAlsoForceRedactionTest() {
+
+        DocumentGraph documentGraph = buildGraph("files/new/crafted document");
+        Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
+
+        RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
+
+        String initialId = entity.getRedactionPositionsPerPage().get(0).getId();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build();
+        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
+                .annotationId(initialId)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.now())
+                .build();
+
+        KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
+        kieSession.insert(entity);
+        kieSession.insert(idRemoval);
+        kieSession.insert(manualForceRedaction);
+        kieSession.fireAllRules();
+        kieSession.dispose();
+
+        assertEquals(ParagraphNode.class, entity.getDeepestFullyContainingNode().getClass());
+        assertFalse(entity.getIntersectingNodes().isEmpty());
+        assertEquals(1, entity.getPages().size());
+        assertEquals("David Ksenia", entity.getValue());
+        assertEquals(initialId, entity.getRedactionPositionsPerPage().get(0).getId());
+        assertFalse(entity.isRemoved());
+    }
+
+
+    @Test
+    public void manualIDRemovalNotApprovedTest() {
+
+        DocumentGraph documentGraph = buildGraph("files/new/crafted document");
+        Set<RedactionEntity> entities = entityCreationService.byKeyword("David Ksenia", "CBI_author", EntityType.ENTITY, documentGraph);
+
+        RedactionEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
+
+        String initialId = entity.getRedactionPositionsPerPage().get(0).getId();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.REQUESTED).build();
+
+        KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
+        kieSession.insert(entity);
+        kieSession.insert(idRemoval);
+        kieSession.fireAllRules();
+        kieSession.dispose();
+
+        assertEquals(ParagraphNode.class, entity.getDeepestFullyContainingNode().getClass());
+        assertFalse(entity.getIntersectingNodes().isEmpty());
+        assertEquals(1, entity.getPages().size());
+        assertEquals("David Ksenia", entity.getValue());
+        assertEquals(initialId, entity.getRedactionPositionsPerPage().get(0).getId());
+        assertFalse(entity.isRemoved());
+    }
+
+
+    private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
+
+        if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
+            return;
+        }
+        // use this for nice formatting of error message
+        assertEquals(rects1, rects2);
+    }
+
+
+    private boolean rectanglesAlmostEqual(Rectangle2D r1, Rectangle2D r2) {
+
+        double tolerance = 1e-1;
+        return Math.abs(r1.getX() - r2.getX()) < tolerance &&//
+                Math.abs(r1.getY() - r2.getY()) < tolerance &&//
+                Math.abs(r1.getWidth() - r2.getWidth()) < tolerance &&//
+                Math.abs(r1.getHeight() - r2.getHeight()) < tolerance;
+    }
+
+
+    private static List<Rectangle> toAnnotationRectangles(RedactionPosition redactionPositions) {
+
+        return redactionPositions.getRectanglePerLine().stream().map(rectangle2D -> toAnnotationRectangle(rectangle2D, redactionPositions.getPageNode().getNumber())).toList();
+    }
+
+
+    private static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, Integer number) {
+
+        return new Rectangle((float) rectangle2D.getMaxX(), (float) rectangle2D.getMaxY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionTest.java

@@ -1,12 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.graph;
-
-import org.junit.jupiter.api.Test;
-
-public class ManualResizeRedactionTest extends BuildDocumentGraphTest {
-
-    @Test
-    public void manualResizeRedactionTest() {
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -103,7 +103,7 @@ public class LiveDataIntegrationTest {
 
     @Configuration
     @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
     public static class RedactionIntegrationTestConfiguration {
 
         @Bean
@@ -124,7 +124,7 @@ public class LiveDataIntegrationTest {
         when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
         when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);
 
-        var rules = IOUtils.toString(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "rules.drl").getInputStream());
+        var rules = IOUtils.toString(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "prod_rules_new.drl").getInputStream());
         when(rulesClient.getRules(any())).thenReturn(JSONPrimitive.of(rules));
 
         ObjectMapper objectMapper = new ObjectMapper();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -39,7 +39,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.ad
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.BlockificationService;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.RulingCleaningService;
@@ -137,7 +137,7 @@ public class PdfSegmentationServiceTest {
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
         assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(table.getColCount()).isEqualTo(6);
         assertThat(table.getRowCount()).isEqualTo(13);
         assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@@ -152,10 +152,10 @@ public class PdfSegmentationServiceTest {
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
         assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(2);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@@ -171,10 +171,10 @@ public class PdfSegmentationServiceTest {
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
         assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(9);
         assertThat(firstTable.getRowCount()).isEqualTo(5);
-        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(9);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
@@ -190,10 +190,10 @@ public class PdfSegmentationServiceTest {
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
         assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/bug_rules.drl

@@ -0,0 +1,8 @@
+import java.util.List
+import com.iqser.red.service.redaction.v1.server.DroolsListBugTest
+
+rule "bugtest"
+    when
+    then
+        DroolsListBugTest.callInFunction();
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -0,0 +1,100 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.exactMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
+
+import java.util.List;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
+import java.util.LinkedList;
+import java.util.HashSet;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import java.util.Set
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
+import java.util.stream.Collectors
+import java.util.Collection
+import java.util.stream.Stream
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
+
+global DocumentGraph document
+global ManualRedactionApplicationService manualRedactionApplicationService
+global Dictionary dictionary
+
+// --------------------------------------- manual redaction rules -------------------------------------------------------------------
+
+rule "Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId)
+        $entityToBeResized: RedactionEntity(matchesAnnotationId($id))
+    then
+        manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+    end
+
+rule "Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.removeFromGraph();
+        retract($entityToBeRemoved);
+    end
+
+rule "Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $imageEntityToBeRemoved: ImageNode($id == id)
+    then
+        $imageEntityToBeRemoved.setIgnored(true);
+    end
+
+rule "Apply force redaction"
+    salience 128
+    when
+        ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
+        $entityToForce: RedactionEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.setLegalBasis($legalBasis);
+        $entityToForce.setRedaction(true);
+        $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
+    end
+
+rule "Apply image recategorization"
+    salience 128
+    when
+        ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
+        $image: ImageNode($id == id)
+    then
+        $image.setImageType(parseImageType($imageType));
+    end
+
+

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_rules_new.drl

@@ -15,6 +15,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import java.util.Set
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
@@ -180,6 +181,15 @@ rule "add NER Entities of type CBI_author or CBI_address"
 
 // --------------------------------------- CBI rules -------------------------------------------------------------------
 
+rule "0: Expand CBI_author and PII matches with salutation prefix"
+    when
+        $entityToExpand: RedactionEntity((type == "CBI_author" || type == "PII"), anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+    then
+        RedactionEntity entity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
+        setFields(entity, 0, null, null, Engine.RULE);
+        insert(entity);
+    end
+
 rule "1: Redact CBI Authors (Non vertebrate study)"
     no-loop true
     when
@@ -438,3 +448,32 @@ rule "18: Redact Phone and Fax by RegEx"
         setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
         entities.forEach(entity -> insert(entity));
     end
+
+// --------------------------------------- Image rules -------------------------------------------------------------------
+
+rule "19: Redact signatures"
+    when
+        $signature: ImageNode(imageType == ImageType.SIGNATURE)
+    then
+        $signature.setRedaction(true);
+        $signature.setRedactionReason("Signature Found");
+        $signature.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+rule "20: Redact formulas"
+    when
+        $signature: ImageNode(imageType == ImageType.FORMULA)
+    then
+        $signature.setRedaction(true);
+        $signature.setRedactionReason("Formula Found");
+        $signature.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+rule "20: Redact logos"
+    when
+        $signature: ImageNode(imageType == ImageType.LOGO)
+    then
+        $signature.setRedaction(true);
+        $signature.setRedactionReason("Logo Found");
+        $signature.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end