Merge branch 'RED-7141' into 'main'

RED-7141: Fixed overlapping blocks

See merge request fforesight/layout-parser!118

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -25,4 +25,5 @@ dependencies {
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
     implementation("org.jgrapht:jgrapht-core:1.5.2")
+    implementation("org.tinspin:tinspin-indexes:2.1.3")
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/NearestNeighbourService.java

@@ -1,10 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
-import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
+import org.tinspin.index.Index;
+import org.tinspin.index.kdtree.KDTree;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;
@@ -13,115 +13,23 @@ import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neigh
 public class NearestNeighbourService {
 
     private static final int NUMBER_OF_NEIGHBOURS = 8;
-    private static final double STEP = 16.0;
 
 
     public void findNearestNeighbors(List<Character> characters) {
+        KDTree<Character> kdTree = KDTree.create(2);
+        characters.forEach(c -> kdTree.insert(new double[]{c.getX(), c.getY()}, c));
 
-        if (characters.isEmpty() || characters.size() == 1) {
-            return;
-        }
-
-        characters.sort(Comparator.comparingDouble(Character::getX));
-
-        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
-        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
-            maxNeighborCount = characters.size() - 1;
-        }
-
-        for (int i = 0; i < characters.size(); i++) {
-
-            Neighbor[] candidates = new Neighbor[maxNeighborCount + 1];
-            int neighborInsertionIndex = 0;
-            int neighborCount = 0;
-            int start = i;
-            int end = i + 1;
-
-            double distance = Double.POSITIVE_INFINITY;
-
-            for (double searchDistance = 0; searchDistance < distance; ) {
-
-                searchDistance += STEP;
-                boolean newCandidatesFound = false;
-
-                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
-                    start--;
-                    candidates[neighborInsertionIndex] = new Neighbor(characters.get(start), characters.get(i));
-                    neighborCount++;
-                    if (neighborCount > maxNeighborCount) {
-                        neighborInsertionIndex = clearMostDistant(candidates);
-                        neighborCount--;
-                    } else {
-                        neighborInsertionIndex++;
-                    }
-                    newCandidatesFound = true;
-                }
-
-                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
-                    candidates[neighborInsertionIndex] = new Neighbor(characters.get(end), characters.get(i));
-                    neighborCount++;
-                    if (neighborCount > maxNeighborCount) {
-                        neighborInsertionIndex = clearMostDistant(candidates);
-                        neighborCount--;
-                    } else {
-                        neighborInsertionIndex++;
-                    }
-                    end++;
-                    newCandidatesFound = true;
-                }
-
-                if (newCandidatesFound && neighborCount >= maxNeighborCount) {
-                    distance = maxDistance(candidates);
-                }
+        for(Character c : characters) {
+            Index.PointIteratorKnn<Character> iterator = kdTree.queryKnn(new double[]{c.getX(), c.getY()}, NUMBER_OF_NEIGHBOURS + 1);
+            // skip the first as this is identity
+            if(iterator.hasNext()) {
+                iterator.next();
             }
-            if (neighborCount < maxNeighborCount) {
-                clearMostDistant(candidates);
-            }
-
-            List<Neighbor> candidatesList = new ArrayList<>(maxNeighborCount);
-            for (Neighbor candidate : candidates) {
-                if (candidate != null) {
-                    candidatesList.add(candidate);
-                }
-            }
-            candidatesList.sort(Comparator.comparingDouble(Neighbor::getDistance));
-            assert candidatesList.size() == maxNeighborCount;
-            characters.get(i).setNeighbors(candidatesList);
-        }
-    }
-
-
-    private double maxDistance(Neighbor[] candidates) {
-
-        double maxDistance = 0;
-        for (Neighbor candidate : candidates) {
-            if (candidate == null) {
-                continue;
-            }
-            if (candidate.getDistance() > maxDistance) {
-                maxDistance = candidate.getDistance();
+            while(iterator.hasNext()) {
+                c.getNeighbors().add(new Neighbor(iterator.next().value(), c));
             }
         }
-        return maxDistance;
-    }
 
-
-    private int clearMostDistant(Neighbor[] candidates) {
-
-        double maxDistance = 0;
-        int maxIndex = 0;
-        for (int i = 0; i < candidates.length; i++) {
-            Neighbor candidate = candidates[i];
-            if (candidate == null) {
-                continue;
-            }
-            if (candidate.getDistance() > maxDistance) {
-                maxDistance = candidate.getDistance();
-                maxIndex = i;
-            }
-        }
-        candidates[maxIndex] = null;
-        return maxIndex;
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ZoneBuilderService.java

@@ -65,7 +65,7 @@ public class ZoneBuilderService {
 
         List<Zone> zones = new ArrayList<>();
         unionFind.getGroups().forEach(group -> {
-            zones.add(new Zone(new ArrayList<>(group)));
+            zones.add(mergeLinesInZone(new ArrayList<>(group), characterSpacing, lineSpacing));
         });
 
         if (zones.size() > MAX_ZONES) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
@@ -23,9 +24,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePag
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
-import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionSequenceComparator;
 
 import lombok.RequiredArgsConstructor;
 
@@ -53,13 +52,13 @@ public class DocstrumBlockificationService {
         });
 
         var zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder);
-        var pageBlocks = toAbstractPageBlocks(zones, usedHorizonalRulings, usedVerticalRulings);
+        var pageBlocks = toAbstractPageBlocks(zones, usedHorizonalRulings, usedVerticalRulings, xyOrder);
 
         return new ClassificationPage(pageBlocks);
     }
 
 
-    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones, List<Ruling> horizontalRulings, List<Ruling> verticalRulings) {
+    private List<AbstractPageBlock> toAbstractPageBlocks(List<Zone> zones, List<Ruling> horizontalRulings, List<Ruling> verticalRulings, boolean xyOrder) {
 
         List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
         zones.forEach(zone -> {
@@ -74,6 +73,18 @@ public class DocstrumBlockificationService {
             abstractPageBlocks.addAll(splitZonesAtRulings(textPositionSequences, horizontalRulings, verticalRulings));
         });
 
+        if (xyOrder) {
+            abstractPageBlocks.sort(Comparator.comparing(AbstractPageBlock::getMinY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(AbstractPageBlock::getMinX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+            abstractPageBlocks.sort(new Comparator<AbstractPageBlock>() {
+                @Override
+                public int compare(AbstractPageBlock o1, AbstractPageBlock o2) {
+
+                    return Math.abs(o1.getMinY() - o2.getMinY()) < 5 && o1.getMinX() < o2.getMinX() == true ? -1 : 0;
+                }
+            });
+        }
+
         return abstractPageBlocks;
     }
 
@@ -88,6 +99,7 @@ public class DocstrumBlockificationService {
 
             AbstractPageBlock block = itty.next();
             if (block instanceof TablePageBlock) {
+                previous = new TextPageBlock();
                 continue;
             }
             TextPageBlock current = (TextPageBlock) block;
@@ -110,12 +122,12 @@ public class DocstrumBlockificationService {
                 }
 
                 if (isSameTopOrBottomWithPreviousHavingMaxTwoLinesAndCurrentThanOneAndMax4OtherBlocksOnHeight(previous, current, page)) {
-                    combineBlocksAndResetIterator(previous, current, itty, false);
+                    combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
                     continue;
                 }
 
                 if (isOnlyIntersectingYAndOnelineOrPrevoiusTwoLines(previous, current, page)) {
-                    combineBlocksAndResetIterator(previous, current, itty, false);
+                    combineBlocksAndResetIterator(previous, current, itty, previous.isToDuplicate());
                     continue;
                 }
 
@@ -140,7 +152,7 @@ public class DocstrumBlockificationService {
                                                                                                               ClassificationPage page) {
 
         return (Math.abs(previous.getMaxY() - current.getMaxY()) < THRESHOLD || Math.abs(previous.getMinY() - current.getMinY()) < THRESHOLD) //
-                && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
+                && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() >= 2 && current.getNumberOfLines() == 1) //
                 && !hasBetween(current, previous, page.getTextBlocks()) && numberOfYIntersectionsOfSmallerBlocksWithOtherBlocks(current, previous, page.getTextBlocks()) <= 4;
     }
 
@@ -215,10 +227,6 @@ public class DocstrumBlockificationService {
 
             TextPageBlock current = (TextPageBlock) block;
 
-            if (current.isToDuplicate()) {
-                continue;
-            }
-
             for (int i = 0; i < blocks.size(); i++) {
 
                 if (toRemove.contains(blocks.get(i))) {
@@ -233,15 +241,12 @@ public class DocstrumBlockificationService {
 
                 TextPageBlock inner = (TextPageBlock) blocks.get(i);
 
-                if (inner.isToDuplicate()) {
-                    continue;
-                }
-
                 if (current.getDir() == inner.getDir() && current.almostIntersects(inner, yThreshold, xThreshold)) {
 
+                    boolean toDuplicate = current.isToDuplicate() || inner.isToDuplicate();
                     current.getSequences().addAll(inner.getSequences());
-                    QuickSort.sort(current.getSequences(), new TextPositionSequenceComparator());
                     current = buildTextBlock(current.getSequences(), 0);
+                    current.setToDuplicate(toDuplicate);
                     toRemove.add(inner);
                     itty.set(current);
                 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -27,7 +27,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
     @SneakyThrows
     public void testViewerDocument() {
 
-        String fileName = "files/bdr/notMergedParagraphs.pdf";
+        String fileName = "files/new/270 rotated text on non rotated pages.pdf";
         String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
 
         var documentFile = new ClassPathResource(fileName).getFile();
@@ -35,7 +35,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
 
         long start = System.currentTimeMillis();
-        Document document = buildGraph(fileName, LayoutParsingType.CLARIFYND);
+        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
         layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }
@@ -59,7 +59,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
                 new ImageServiceResponse(),
                 tableResponse,
                 new VisualLayoutParsingResponse(),
-                Map.of("file",Path.of(fileName).getFileName().toFile().toString()));
+                Map.of("file", Path.of(fileName).getFileName().toFile().toString()));
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
         Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE, classificationDocument);