Merge branch 'clari-30' into 'main'

CLARI-30 - identifier fix for clarifynd

See merge request fforesight/layout-parser!111

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -118,7 +118,7 @@ public class LayoutParsingPipeline {
                 imageServiceResponse,
                 tableServiceResponse,
                 visualLayoutParsingResponse,
-                layoutParsingRequest.identifier().toString());
+                layoutParsingRequest.identifier());
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
@@ -207,7 +207,7 @@ public class LayoutParsingPipeline {
                                               ImageServiceResponse imageServiceResponse,
                                               TableServiceResponse tableServiceResponse,
                                               VisualLayoutParsingResponse visualLayoutParsingResponse,
-                                              String identifier) {
+                                              Map<String, String> identifier) {
 
         PDDocument originDocument = openDocument(originFile);
         addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
@@ -291,7 +291,7 @@ public class LayoutParsingPipeline {
             if (layoutParsingType == LayoutParsingType.REDACT_MANAGER) {
                 docstrumBlockificationService.combineBlocks(classificationPage);
             } else if (layoutParsingType == LayoutParsingType.CLARIFYND) {
-                docstrumBlockificationService.mergeZones(classificationPage.getTextBlocks());
+                docstrumBlockificationService.mergeIntersectingBlocks(classificationPage.getTextBlocks());
             }
 
             buildPageStatistics(classificationPage);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/DocstrumSegmentationService.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -6,15 +6,15 @@ import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.LineBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.NearestNeighbourService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.ReadingOrderService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.SpacingService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.ZoneBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.LineBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.NearestNeighbourService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ReadingOrderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.SpacingService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ZoneBuilderService;
 
 import lombok.RequiredArgsConstructor;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/AngleFilter.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
 public class AngleFilter {
 
@@ -7,7 +7,7 @@ public class AngleFilter {
 
 
     public AngleFilter(double lowerAngle, double upperAngle) {
-        
+
         this.lowerAngle = lowerAngle < -Math.PI / 2 ? lowerAngle + Math.PI : lowerAngle;
         this.upperAngle = upperAngle >= Math.PI / 2 ? upperAngle - Math.PI : upperAngle;
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/BoundingBox.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
 import java.awt.geom.Rectangle2D;
 
@@ -42,10 +42,7 @@ public abstract class BoundingBox {
 
     public boolean contains(Rectangle2D contained, double tolerance) {
 
-        return bBox.getX() <= contained.getX() + tolerance
-               && bBox.getY() <= contained.getY() + tolerance
-               && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance
-               && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
+        return bBox.getX() <= contained.getX() + tolerance && bBox.getY() <= contained.getY() + tolerance && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Character.java

@@ -1,20 +1,27 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.Setter;
 
 @Data
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Character {
 
+    @EqualsAndHashCode.Include
     private final double x;
+    @EqualsAndHashCode.Include
     private final double y;
     private final RedTextPosition textPosition;
 
+    @Setter
     private List<Neighbor> neighbors = new ArrayList<>();
 
 
@@ -67,18 +74,12 @@ public class Character {
     }
 
 
-    public void setNeighbors(List<Neighbor> neighbors) {
-
-        this.neighbors = neighbors;
-    }
-
-
     public double angle(Character character) {
 
         if (getX() > character.getX()) {
-            return Math.atan2(getY() - character.getY(), getX() - character.getX());
+            return FastAtan2.fastAtan2(getY() - character.getY(), getX() - character.getX());
         } else {
-            return Math.atan2(character.getY() - getY(), character.getX() - getX());
+            return FastAtan2.fastAtan2(character.getY() - getY(), character.getX() - getX());
         }
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Histogram.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
 public class Histogram {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
@@ -8,16 +8,22 @@ import java.util.List;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 
 @Data
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Line extends BoundingBox {
 
     private static final double WORD_DISTANCE_MULTIPLIER = 0.18;
 
+    @EqualsAndHashCode.Include
     private final double x0;
+    @EqualsAndHashCode.Include
     private final double y0;
 
+    @EqualsAndHashCode.Include
     private final double x1;
+    @EqualsAndHashCode.Include
     private final double y1;
 
     private final double height;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Neighbor.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
 import lombok.Getter;
 
@@ -6,8 +6,7 @@ public class Neighbor {
 
     @Getter
     private final double distance;
-    @Getter
-    private final double angle;
+    private Double angle;
     private final Character originCharacter;
     @Getter
     private final Character character;
@@ -16,7 +15,6 @@ public class Neighbor {
     public Neighbor(Character neighbor, Character origin) {
 
         this.distance = neighbor.distance(origin);
-        this.angle = neighbor.angle(origin);
         this.character = neighbor;
         this.originCharacter = origin;
     }
@@ -33,4 +31,13 @@ public class Neighbor {
         return character.verticalDistance(originCharacter);
     }
 
+
+    public double getAngle() {
+
+        if (angle != null) {
+            return angle;
+        }
+        return this.character.angle(this.originCharacter);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/UnionFind.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
 import java.util.Collection;
 import java.util.LinkedHashMap;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Zone.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Comparator;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/LineBuilderService.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -7,10 +7,10 @@ import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
 
 @Service
 public class LineBuilderService {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/NearestNeighbourService.java

@@ -0,0 +1,128 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;
+
+@Service
+public class NearestNeighbourService {
+
+    private static final int NUMBER_OF_NEIGHBOURS = 8;
+    private static final double STEP = 16.0;
+
+
+    public void findNearestNeighbors(List<Character> characters) {
+
+        if (characters.isEmpty() || characters.size() == 1) {
+            return;
+        }
+
+        characters.sort(Comparator.comparingDouble(Character::getX));
+
+        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
+        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
+            maxNeighborCount = characters.size() - 1;
+        }
+
+        for (int i = 0; i < characters.size(); i++) {
+
+            Neighbor[] candidates = new Neighbor[maxNeighborCount + 1];
+            int neighborInsertionIndex = 0;
+            int neighborCount = 0;
+            int start = i;
+            int end = i + 1;
+
+            double distance = Double.POSITIVE_INFINITY;
+
+            for (double searchDistance = 0; searchDistance < distance; ) {
+
+                searchDistance += STEP;
+                boolean newCandidatesFound = false;
+
+                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
+                    start--;
+                    candidates[neighborInsertionIndex] = new Neighbor(characters.get(start), characters.get(i));
+                    neighborCount++;
+                    if (neighborCount > maxNeighborCount) {
+                        neighborInsertionIndex = clearMostDistant(candidates);
+                        neighborCount--;
+                    } else {
+                        neighborInsertionIndex++;
+                    }
+                    newCandidatesFound = true;
+                }
+
+                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
+                    candidates[neighborInsertionIndex] = new Neighbor(characters.get(end), characters.get(i));
+                    neighborCount++;
+                    if (neighborCount > maxNeighborCount) {
+                        neighborInsertionIndex = clearMostDistant(candidates);
+                        neighborCount--;
+                    } else {
+                        neighborInsertionIndex++;
+                    }
+                    end++;
+                    newCandidatesFound = true;
+                }
+
+                if (newCandidatesFound && neighborCount >= maxNeighborCount) {
+                    distance = maxDistance(candidates);
+                }
+            }
+            if (neighborCount < maxNeighborCount) {
+                clearMostDistant(candidates);
+            }
+
+            List<Neighbor> candidatesList = new ArrayList<>(maxNeighborCount);
+            for (Neighbor candidate : candidates) {
+                if (candidate != null) {
+                    candidatesList.add(candidate);
+                }
+            }
+            candidatesList.sort(Comparator.comparingDouble(Neighbor::getDistance));
+            assert candidatesList.size() == maxNeighborCount;
+            characters.get(i).setNeighbors(candidatesList);
+        }
+    }
+
+
+    private double maxDistance(Neighbor[] candidates) {
+
+        double maxDistance = 0;
+        for (Neighbor candidate : candidates) {
+            if (candidate == null) {
+                continue;
+            }
+            if (candidate.getDistance() > maxDistance) {
+                maxDistance = candidate.getDistance();
+            }
+        }
+        return maxDistance;
+    }
+
+
+    private int clearMostDistant(Neighbor[] candidates) {
+
+        double maxDistance = 0;
+        int maxIndex = 0;
+        for (int i = 0; i < candidates.length; i++) {
+            Neighbor candidate = candidates[i];
+            if (candidate == null) {
+                continue;
+            }
+            if (candidate.getDistance() > maxDistance) {
+                maxDistance = candidate.getDistance();
+                maxIndex = i;
+            }
+        }
+        candidates[maxIndex] = null;
+        return maxIndex;
+    }
+
+}
+

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ReadingOrderService.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -9,9 +9,9 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 
 @Service
 public class ReadingOrderService {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/SpacingService.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
 import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Histogram;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Histogram;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;
 
 @Service
 public class SpacingService {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ZoneBuilderService.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -8,10 +8,10 @@ import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.UnionFind;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 
 @Service
 public class ZoneBuilderService {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/utils/DoubleUtils.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils;
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.utils;
 
 public class DoubleUtils {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/utils/FastAtan2.java

@@ -0,0 +1,76 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.utils;
+
+public class FastAtan2 {
+
+    static final private int Size_Ac = 1000;
+    static final private int Size_Ar = Size_Ac + 1;
+    static final private double Pi = (float) Math.PI;
+    static final private double Pi_H = Pi / 2;
+
+    static final private double[] Atan2 = new double[Size_Ar];
+    static final private double[] Atan2_PM = new double[Size_Ar];
+    static final private double[] Atan2_MP = new double[Size_Ar];
+    static final private double[] Atan2_MM = new double[Size_Ar];
+
+    static final private double[] Atan2_R = new double[Size_Ar];
+    static final private double[] Atan2_RPM = new double[Size_Ar];
+    static final private double[] Atan2_RMP = new double[Size_Ar];
+    static final private double[] Atan2_RMM = new double[Size_Ar];
+
+    static {
+        for (int i = 0; i <= Size_Ac; i++) {
+            double d = (double) i / Size_Ac;
+            double x = 1;
+            double y = x * d;
+            double v = Math.atan2(y, x);
+            Atan2[i] = v;
+            Atan2_PM[i] = Pi - v;
+            Atan2_MP[i] = -v;
+            Atan2_MM[i] = -Pi + v;
+
+            Atan2_R[i] = Pi_H - v;
+            Atan2_RPM[i] = Pi_H + v;
+            Atan2_RMP[i] = -Pi_H + v;
+            Atan2_RMM[i] = -Pi_H - v;
+
+        }
+    }
+
+    @SuppressWarnings("ParameterAssignment")
+    static public double fastAtan2(double y, double x) {
+
+        if (y < 0) {
+            if (x < 0) {
+                //(y < x) because == (-y > -x)
+                if (y < x) {
+                    return Atan2_RMM[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_MM[(int) (y / x * Size_Ac)];
+                }
+            } else {
+                y = -y;
+                if (y > x) {
+                    return Atan2_RMP[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_MP[(int) (y / x * Size_Ac)];
+                }
+            }
+        } else {
+            if (x < 0) {
+                x = -x;
+                if (y > x) {
+                    return Atan2_RPM[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_PM[(int) (y / x * Size_Ac)];
+                }
+            } else {
+                if (y > x) {
+                    return Atan2_R[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2[(int) (y / x * Size_Ac)];
+                }
+            }
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import org.apache.pdfbox.text.TextPosition;
-import org.springframework.beans.BeanUtils;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
@@ -50,10 +49,14 @@ public class RedTextPosition {
     public static RedTextPosition fromTextPosition(TextPosition textPosition) {
 
         var pos = new RedTextPosition();
-        BeanUtils.copyProperties(textPosition, pos);
-        pos.setFontName(textPosition.getFont().getName());
-
+        pos.setRotation(textPosition.getRotation());
+        pos.setPageHeight(textPosition.getPageHeight());
+        pos.setPageWidth(textPosition.getPageWidth());
+        pos.setUnicode(textPosition.getUnicode());
+        pos.setDir(textPosition.getDir());
+        pos.setWidthOfSpace(textPosition.getWidthOfSpace());
         pos.setFontSizeInPt(textPosition.getFontSizeInPt());
+        pos.setFontName(textPosition.getFont().getName());
 
         var position = new float[4];
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -24,12 +25,18 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TextPositionSequence implements CharSequence {
 
     public static final int HEIGHT_PADDING = 2;
+
+    @EqualsAndHashCode.Include
     private int page;
+
+    @EqualsAndHashCode.Include
     private List<RedTextPosition> textPositions = new ArrayList<>();
 
+    @EqualsAndHashCode.Include
     private TextDirection dir;
     private int rotation;
     private float pageHeight;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -12,6 +12,8 @@ import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.DocstrumSegmentationService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
@@ -21,7 +23,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePag
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionSequenceComparator;
@@ -51,8 +52,16 @@ public class DocstrumBlockificationService {
             usedVerticalRulings.add(new Ruling(new Point2D.Float(cell.x + cell.width, cell.y), new Point2D.Float(cell.x + cell.width, cell.y + cell.height)));
         });
 
-        List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
         var zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder);
+        var pageBlocks = toAbstractPageBlocks(zones, usedHorizonalRulings, usedVerticalRulings);
+
+        return new ClassificationPage(pageBlocks);
+    }
+
+
+    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones, List<Ruling> horizontalRulings, List<Ruling> verticalRulings) {
+
+        List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
         zones.forEach(zone -> {
 
             List<TextPositionSequence> textPositionSequences = new ArrayList<>();
@@ -62,20 +71,21 @@ public class DocstrumBlockificationService {
                 });
             });
 
-            abstractPageBlocks.addAll(splitZonesAtRulings(textPositionSequences, usedHorizonalRulings, usedVerticalRulings));
+            abstractPageBlocks.addAll(splitZonesAtRulings(textPositionSequences, horizontalRulings, verticalRulings));
         });
 
-        return new ClassificationPage(abstractPageBlocks);
+        return abstractPageBlocks;
     }
 
 
     public void combineBlocks(ClassificationPage page) {
 
-        mergeZones(page.getTextBlocks());
+        mergeIntersectingBlocks(page.getTextBlocks());
 
         TextPageBlock previous = new TextPageBlock();
         ListIterator<AbstractPageBlock> itty = page.getTextBlocks().listIterator();
         while (itty.hasNext()) {
+
             AbstractPageBlock block = itty.next();
             if (block instanceof TablePageBlock) {
                 continue;
@@ -84,60 +94,28 @@ public class DocstrumBlockificationService {
 
             if (previous != null && !previous.getSequences().isEmpty()) {
 
-                if (current.getDir() == previous.getDir() //
-                        && previous.getNumberOfLines() >= 2 && current.getNumberOfLines() >= 2 //
-                        && previous.intersectsY(current) //
-                        && !hasBetween(current, previous, page.getTextBlocks()) //
-                        && numberOfYIntersections(current, previous, page.getTextBlocks()) == 0) {
-
-                    previous.getSequences().addAll(current.getSequences());
-                    previous = buildTextBlock(previous.getSequences(), 0);
-                    previous.setToDuplicate(true);
-                    itty.remove();
-                    itty.previous();
-                    itty.set(previous);
-                    itty.next();
+                if (current.getDir() != previous.getDir()) {
+                    previous = current;
                     continue;
                 }
 
-                if (current.getDir() == previous.getDir() && (previous.almostIntersects(current, 0, 0))) {
-
-                    previous.getSequences().addAll(current.getSequences());
-                    boolean toDuplicate = previous.isToDuplicate();
-                    previous = buildTextBlock(previous.getSequences(), 0);
-                    previous.setToDuplicate(toDuplicate);
-                    itty.remove();
-                    itty.previous();
-                    itty.set(previous);
-                    itty.next();
+                if (areTheOnlyTwoBlocksOnHeightsWithBothMoreThanTwoLines(previous, current, page)) {
+                    combineBlocksAndResetIterator(previous, current, itty, true);
                     continue;
                 }
 
-                if (current.getDir() == previous.getDir() //
-                        && (Math.abs(previous.getMaxY() - current.getMaxY()) < THRESHOLD || Math.abs(previous.getMinY() - current.getMinY()) < THRESHOLD) //
-                        && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
-                        && !hasBetween(current, previous, page.getTextBlocks()) && numberOfYIntersections(current, previous, page.getTextBlocks()) <= 4) {
-
-                    previous.getSequences().addAll(current.getSequences());
-                    previous = buildTextBlock(previous.getSequences(), 0);
-                    itty.remove();
-                    itty.previous();
-                    itty.set(previous);
-                    itty.next();
+                if (previous.almostIntersects(current, 0, 0)) {
+                    combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
                     continue;
                 }
 
-                if (current.getDir() == previous.getDir() //
-                        && current.intersectsY(previous) //
-                        && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
-                        && !hasBetween(current, previous, page.getTextBlocks()) //
-                        && numberOfYIntersections(current, previous, page.getTextBlocks()) <= 0) {
-                    previous.getSequences().addAll(current.getSequences());
-                    previous = buildTextBlock(previous.getSequences(), 0);
-                    itty.remove();
-                    itty.previous();
-                    itty.set(previous);
-                    itty.next();
+                if (isSameTopOrBottomWithPreviousHavingMaxTwoLinesAndCurrentThanOneAndMax4OtherBlocksOnHeight(previous, current, page)) {
+                    combineBlocksAndResetIterator(previous, current, itty, false);
+                    continue;
+                }
+
+                if (isOnlyIntersectingYAndOnelineOrPrevoiusTwoLines(previous, current, page)) {
+                    combineBlocksAndResetIterator(previous, current, itty, false);
                     continue;
                 }
 
@@ -145,8 +123,45 @@ public class DocstrumBlockificationService {
             previous = current;
         }
 
-        mergeZones(page.getTextBlocks());
+        mergeIntersectingBlocks(page.getTextBlocks());
+    }
 
+
+    private boolean isOnlyIntersectingYAndOnelineOrPrevoiusTwoLines(TextPageBlock previous, TextPageBlock current, ClassificationPage page) {
+
+        return current.intersectsY(previous) //
+                && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
+                && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) <= 0;
+    }
+
+
+    private boolean isSameTopOrBottomWithPreviousHavingMaxTwoLinesAndCurrentThanOneAndMax4OtherBlocksOnHeight(TextPageBlock previous,
+                                                                                                              TextPageBlock current,
+                                                                                                              ClassificationPage page) {
+
+        return (Math.abs(previous.getMaxY() - current.getMaxY()) < THRESHOLD || Math.abs(previous.getMinY() - current.getMinY()) < THRESHOLD) //
+                && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
+                && !hasBetween(current, previous, page.getTextBlocks()) && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) <= 4;
+    }
+
+
+    private boolean areTheOnlyTwoBlocksOnHeightsWithBothMoreThanTwoLines(TextPageBlock previous, TextPageBlock current, ClassificationPage page) {
+
+        return previous.getNumberOfLines() >= 2 && current.getNumberOfLines() >= 2 //
+                && previous.intersectsY(current) //
+                && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) == 0;
+    }
+
+
+    private void combineBlocksAndResetIterator(TextPageBlock previous, TextPageBlock current, ListIterator<AbstractPageBlock> itty, boolean toDuplicate) {
+
+        previous.getSequences().addAll(current.getSequences());
+        previous = buildTextBlock(previous.getSequences(), 0);
+        previous.setToDuplicate(toDuplicate);
+        itty.remove();
+        itty.previous();
+        itty.set(previous);
+        itty.next();
     }
 
 
@@ -167,7 +182,7 @@ public class DocstrumBlockificationService {
     }
 
 
-    private int numberOfYIntersections(TextPageBlock block, TextPageBlock other, List<AbstractPageBlock> allBlocks) {
+    private int numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(TextPageBlock block, TextPageBlock other, List<AbstractPageBlock> allBlocks) {
 
         double minY = Math.min(block.getMinY(), other.getMinY());
         double maxY = Math.min(block.getMaxY(), other.getMaxY());
@@ -188,9 +203,9 @@ public class DocstrumBlockificationService {
     }
 
 
-    public void mergeZones(List<AbstractPageBlock> zones) {
+    public void mergeIntersectingBlocks(List<AbstractPageBlock> blocks) {
 
-        ListIterator<AbstractPageBlock> itty = zones.listIterator();
+        ListIterator<AbstractPageBlock> itty = blocks.listIterator();
         Set<AbstractPageBlock> toRemove = new HashSet<>();
         while (itty.hasNext()) {
             AbstractPageBlock block = itty.next();
@@ -204,19 +219,19 @@ public class DocstrumBlockificationService {
                 continue;
             }
 
-            for (int i = 0; i < zones.size(); i++) {
+            for (int i = 0; i < blocks.size(); i++) {
 
-                if (toRemove.contains(zones.get(i))) {
+                if (toRemove.contains(blocks.get(i))) {
                     continue;
                 }
-                if (zones.get(i) == current) {
+                if (blocks.get(i) == current) {
                     continue;
                 }
-                if (zones.get(i) instanceof TablePageBlock) {
+                if (blocks.get(i) instanceof TablePageBlock) {
                     continue;
                 }
 
-                TextPageBlock inner = (TextPageBlock) zones.get(i);
+                TextPageBlock inner = (TextPageBlock) blocks.get(i);
 
                 if (inner.isToDuplicate()) {
                     continue;
@@ -232,7 +247,7 @@ public class DocstrumBlockificationService {
                 }
             }
         }
-        zones.removeAll(toRemove);
+        blocks.removeAll(toRemove);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/NearestNeighbourService.java

@@ -1,78 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
-
-@Service
-public class NearestNeighbourService {
-
-    private static final int NUMBER_OF_NEIGHBOURS = 8;
-    private static final double STEP = 16.0;
-
-
-    public void findNearestNeighbors(List<Character> characters) {
-
-        if (characters.isEmpty() || characters.size() == 1) {
-            return;
-        }
-
-        characters.sort(Comparator.comparingDouble(Character::getX));
-
-        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
-        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
-            maxNeighborCount = characters.size() - 1;
-        }
-
-        for (int i = 0; i < characters.size(); i++) {
-
-            List<Neighbor> candidates = new ArrayList<>();
-
-            int start = i;
-            int end = i + 1;
-
-            double distance = Double.POSITIVE_INFINITY;
-
-            for (double searchDistance = 0; searchDistance < distance; ) {
-
-                searchDistance += STEP;
-                boolean newCandidatesFound = false;
-
-                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
-                    start--;
-                    candidates.add(new Neighbor(characters.get(start), characters.get(i)));
-                    clearLeastDistant(candidates, maxNeighborCount);
-                    newCandidatesFound = true;
-                }
-
-                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
-                    candidates.add(new Neighbor(characters.get(end), characters.get(i)));
-                    clearLeastDistant(candidates, maxNeighborCount);
-                    end++;
-                    newCandidatesFound = true;
-                }
-
-                if (newCandidatesFound && candidates.size() >= maxNeighborCount) {
-                    distance = candidates.get(maxNeighborCount - 1).getDistance();
-                }
-            }
-            clearLeastDistant(candidates, maxNeighborCount);
-            characters.get(i).setNeighbors(new ArrayList<>(candidates));
-        }
-    }
-
-
-    private void clearLeastDistant(List<Neighbor> candidates, int maxNeighborCount) {
-
-        if (candidates.size() > maxNeighborCount) {
-            candidates.sort(Comparator.comparingDouble(Neighbor::getDistance));
-            candidates.remove(candidates.remove(candidates.size() - 1));
-        }
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/RectangleTransformations.java

@@ -2,7 +2,6 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 
 import static java.lang.String.format;
 
-import java.awt.geom.Area;
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
 import java.util.Collections;
@@ -40,11 +39,10 @@ public class RectangleTransformations {
 
     public static double calculateIntersectedArea(Rectangle2D r1, Rectangle2D r2) {
 
-        Area a1 = new Area(r1);
-        Area a2 = new Area(r2);
-        a1.intersect(a2);
-        Rectangle2D intersection = a1.getBounds2D();
-        return intersection.getWidth() * intersection.getHeight();
+        double xOverlap = Math.max(0, Math.min(r1.getMaxX(), r2.getMaxX()) - Math.max(r1.getMinX(), r2.getMinX()));
+        double yOverlap = Math.max(0, Math.min(r1.getMaxY(), r2.getMaxY()) - Math.max(r1.getY(), r2.getY()));
+
+        return xOverlap * yOverlap;
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -11,6 +11,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -54,7 +55,7 @@ public class BdrJsonBuildTest extends AbstractTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        file.toString()));
+                        Map.of("file",file.toString())));
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.server;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -101,7 +102,7 @@ public class HeadlinesGoldStandardIntegrationTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filePath));
+                        Map.of("file",filePath)));
 
         var foundHeadlines = documentGraph.streamAllSubNodes()
                 .map(SemanticNode::getHeadline)

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphJsonWritingTest.java

@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -61,7 +62,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
 
         DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
         ObjectMapper mapper = ObjectMapperFactory.create();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.server.graph;
 
 import java.io.File;
 import java.nio.file.Path;
+import java.util.Map;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -33,8 +34,8 @@ public class ViewerDocumentTest extends BuildDocumentTest {
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
 
-        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
         long start = System.currentTimeMillis();
+        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
         layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }
@@ -58,7 +59,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
                 new ImageServiceResponse(),
                 tableResponse,
                 new VisualLayoutParsingResponse(),
-                Path.of(fileName).getFileName().toFile().toString());
+                Map.of("file",Path.of(fileName).getFileName().toFile().toString()));
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
         Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE, classificationDocument);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -61,7 +61,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                 new ImageServiceResponse(),
                 tableServiceResponse,
                 new VisualLayoutParsingResponse(),
-                "document");
+                Map.of("file","document"));
 
         redactManagerClassificationService.classifyDocument(classificationDocument);
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -7,6 +7,7 @@ import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.apache.pdfbox.Loader;
@@ -105,14 +106,14 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
         Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
                 layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
                         filename.toFile(),
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
         DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
         DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
         if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure())) {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;
 
 import java.io.File;
+import java.util.Map;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
@@ -31,7 +32,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
                 layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
                 new TableServiceResponse(),
                 new VisualLayoutParsingResponse(),
-                filename);
+                Map.of("file",filename));
     }