RED-6009 - Document Tree Structure

* added gaps in Redaction Positions, if they are farther than five character widths apart

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java

@@ -137,7 +137,7 @@ public class RedactionEntity {
     public List<RedactionPosition> getRedactionPositionsPerPage() {
 
         if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) {
-            Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.buildTextBlock().getEntityPositionsPerPage(boundary);
+            Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.buildTextBlock().getPositionsPerPage(boundary);
 
             Page firstPage = rectanglesPerLinePerPage.keySet()
                     .stream()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java

@@ -379,7 +379,7 @@ public interface SemanticNode {
 
 
     /**
-     * TODO this does not yet work for sections spanning multiple columns.
+     * TODO: this produces unwanted results for sections spanning multiple columns.
      *
      * @param bBoxPerPage initial empty BoundingBox
      * @return The union of the BoundingBoxes of all children

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.t
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -119,12 +120,13 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    public Map<Page, List<Rectangle2D>> getEntityPositionsPerPage(Boundary stringBoundary) {
+    public Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary) {
 
-        List<Rectangle2D> rectanglesPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList())
+        List<Rectangle2D> rectanglesPerLine = stringBoundary.split(getAllLineBreaksInBoundary(stringBoundary))
                 .stream()
                 .map(this::getPositions)
-                .map(RectangleTransformations::rectangleUnion)
+                .map(RectangleTransformations::rectangleUnionWithGaps)
+                .flatMap(Collection::stream)
                 .toList();
         Map<Page, List<Rectangle2D>> rectanglePerLinePerPage = new HashMap<>();
         rectanglePerLinePerPage.put(page, rectanglesPerLine);
@@ -132,6 +134,12 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
+    private List<Integer> getAllLineBreaksInBoundary(Boundary boundary) {
+
+        return getLineBreaks().stream().map(linebreak -> linebreak + this.boundary.start()).filter(boundary::contains).toList();
+    }
+
+
     @Override
     public String toString() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java

@@ -137,23 +137,23 @@ public class ConcatenatedTextBlock implements TextBlock {
 
 
     @Override
-    public Map<Page, List<Rectangle2D>> getEntityPositionsPerPage(Boundary stringBoundary) {
+    public Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary) {
 
         List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
 
         if (textBlocks.size() == 1) {
-            return textBlocks.get(0).getEntityPositionsPerPage(stringBoundary);
+            return textBlocks.get(0).getPositionsPerPage(stringBoundary);
         }
 
         AtomicTextBlock firstTextBlock = textBlocks.get(0);
-        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = firstTextBlock.getEntityPositionsPerPage(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end()));
+        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = firstTextBlock.getPositionsPerPage(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end()));
 
         for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
-            rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, textBlock.getEntityPositionsPerPage(textBlock.getBoundary()));
+            rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, textBlock.getPositionsPerPage(textBlock.getBoundary()));
         }
 
         AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
-        rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, lastTextBlock.getEntityPositionsPerPage(lastTextBlock.getBoundary()));
+        rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, lastTextBlock.getPositionsPerPage(lastTextBlock.getBoundary()));
 
         return rectanglesPerLinePerPage;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlock.java

@@ -39,7 +39,7 @@ public interface TextBlock extends CharSequence {
     List<Rectangle2D> getPositions(Boundary stringBoundary);
 
 
-    Map<Page, List<Rectangle2D>> getEntityPositionsPerPage(Boundary stringBoundary);
+    Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary);
 
 
     int numberOfLines();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java

@@ -2,6 +2,9 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
 
 import java.awt.geom.Area;
 import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
+import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 import java.util.function.BiConsumer;
@@ -61,6 +64,38 @@ public class RectangleTransformations {
     }
 
 
+    /**
+     * If two rectangles are further apart than five times the average width of a rectangle, a gap is inserted.
+     *
+     * @param rectangle2DList A list of rectangles to combine
+     * @return A list of rectangles which are combined if they are closer than the split threshold
+     */
+    public static List<Rectangle2D> rectangleUnionWithGaps(List<Rectangle2D> rectangle2DList) {
+
+        if (rectangle2DList.isEmpty()) {
+            return Collections.emptyList();
+        }
+        double split_threshold = rectangle2DList.stream().mapToDouble(RectangularShape::getWidth).average().orElse(5) * 5.0;
+
+        List<List<Rectangle2D>> rectangleListsWithGaps = new LinkedList<>();
+        List<Rectangle2D> rectangleListWithoutGaps = new LinkedList<>();
+        rectangleListsWithGaps.add(rectangleListWithoutGaps);
+        Rectangle2D previousRectangle = rectangle2DList.get(0);
+        for (Rectangle2D currentRectangle : rectangle2DList) {
+            if (Math.abs(currentRectangle.getMinX() - previousRectangle.getMaxX()) > split_threshold) {
+                rectangleListWithoutGaps = new LinkedList<>();
+                rectangleListWithoutGaps.add(currentRectangle);
+                rectangleListsWithGaps.add(rectangleListWithoutGaps);
+                previousRectangle = currentRectangle;
+            } else {
+                rectangleListWithoutGaps.add(currentRectangle);
+                previousRectangle = currentRectangle;
+            }
+        }
+        return rectangleListsWithGaps.stream().map(RectangleTransformations::rectangleUnion).toList();
+    }
+
+
     private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RuleBuilderModelService.java

@@ -9,6 +9,7 @@ import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
 @Service
 public class RuleBuilderModelService {
 
+    // TODO: Evaluate if we need to implement this or remove it entirely.
     public RuleBuilderModel getRuleBuilderModel() {
 
         RuleBuilderModel ruleBuilderModel = new RuleBuilderModel();