Merge branch 'RED-7384' into 'release/0.89.x'

RED-7384: add empty textBlock to Image to ensure continuous textranges across all SemanticNodes

See merge request fforesight/layout-parser!139

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -28,6 +28,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
@@ -250,8 +251,10 @@ public class LayoutParsingPipeline {
             PDRectangle cropbox = pdPage.getCropBox();
             CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
 
+            List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
+
             ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells);
                 case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
                 case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
             };

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java

@@ -12,7 +12,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.No
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -36,6 +35,8 @@ public class Image implements GenericSemanticNode {
     boolean transparent;
     Rectangle2D position;
 
+    TextBlock leafTextBlock;
+
     boolean redaction;
     boolean ignored;
     @Builder.Default
@@ -66,7 +67,7 @@ public class Image implements GenericSemanticNode {
     @Override
     public TextBlock getTextBlock() {
 
-        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return leafTextBlock;
     }
 
 
@@ -92,4 +93,11 @@ public class Image implements GenericSemanticNode {
         return bBoxPerPage;
     }
 
+
+    @Override
+    public boolean isLeaf() {
+
+        return true;
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionsBuilderService.java

@@ -43,7 +43,6 @@ public class SectionsBuilderService {
         for (ClassificationPage page : document.getPages()) {
             List<TextPageBlock> header = new ArrayList<>();
             List<TextPageBlock> footer = new ArrayList<>();
-            List<TextPageBlock> unclassifiedText = new ArrayList<>();
             for (AbstractPageBlock current : page.getTextBlocks()) {
 
                 if (current.getClassification() == null) {
@@ -62,11 +61,6 @@ public class SectionsBuilderService {
                     continue;
                 }
 
-                if (current.getClassification().equals(PageBlockType.OTHER)) {
-                    unclassifiedText.add((TextPageBlock) current);
-                    continue;
-                }
-
                 if (prev != null && current.getClassification().isHeadline() && !prev.getClassification().isHeadline() || !document.isHeadlines()) {
                     ClassificationSection chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
@@ -94,9 +88,6 @@ public class SectionsBuilderService {
             if (!footer.isEmpty()) {
                 footers.add(new ClassificationFooter(footer));
             }
-            if (!unclassifiedText.isEmpty()) {
-                unclassifiedTexts.add(new UnclassifiedText(unclassifiedText));
-            }
         }
 
         ClassificationSection chunkBlock = buildTextBlock(chunkWords, lastHeadline);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.blockifica
 
 import static java.util.stream.Collectors.toSet;
 
+import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -13,10 +14,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
 
 @SuppressWarnings("all")
@@ -31,12 +35,12 @@ public class RedactManagerBlockificationService {
      * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
      * Rulings (TablePageBlock lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
      *
-     * @param textPositions         The words of a page.
-     * @param horizontalRulingLines Horizontal table lines.
-     * @param verticalRulingLines   Vertical table lines.
+     * @param textPositions The words of a page.
      * @return Page object that contains the Textblock and text statistics.
      */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Cell> emptyCells) {
+
+        CleanRulings usedRulings = RectangleTransformations.extractRulings(emptyCells);
 
         int indexOnPage = 0;
         List<TextPositionSequence> chunkWords = new ArrayList<>();
@@ -54,7 +58,7 @@ public class RedactManagerBlockificationService {
             boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
             boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
-            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
+            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, usedRulings.getHorizontal(), usedRulings.getVertical());
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
 
             if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java

@@ -18,8 +18,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
@@ -31,6 +29,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
@@ -99,19 +99,12 @@ public class DocumentGraphFactory {
     }
 
 
-    public void addImage(Section section, ClassifiedImage image, Context context) {
+    public void addImage(GenericSemanticNode parent, ClassifiedImage image, Context context) {
 
         Image imageNode = createImage(image, context);
-        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode);
-        imageNode.setTreeId(treeId);
-    }
-
-
-    public void addImage(Document document, ClassifiedImage image, Context context) {
-
-        Image imageNode = createImage(image, context);
-        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(document, imageNode);
+        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parent, imageNode);
         imageNode.setTreeId(treeId);
+        imageNode.setLeafTextBlock(context.textBlockFactory.emptyTextBlock(parent, context, context.getPage(image.getPage())));
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java

@@ -110,11 +110,13 @@ public class LayoutGridService {
             return;
         }
         for (Page page : table.getPages()) {
+
             Optional<Integer> optionalFirstRowOnPage = table.streamCol(0).filter(tableCell -> tableCell.isOnPage(page.getNumber())).map(TableCell::getRow).findFirst();
             if (optionalFirstRowOnPage.isEmpty()) {
                 continue;
             }
             int firstRowOnPage = optionalFirstRowOnPage.get();
+
             Stream<Double> xStream = switch (page.getRotation()) {
                 case 90 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinX);
                 case 180 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMaxX);
@@ -123,6 +125,7 @@ public class LayoutGridService {
             };
             List<Double> xs = xStream.collect(Collectors.toList());
             xs.remove(0);
+
             Stream<Double> yStream = switch (page.getRotation()) {
                 case 90 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMinY);
                 case 180 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinY);
@@ -132,7 +135,7 @@ public class LayoutGridService {
             List<Double> ys = yStream.collect(Collectors.toList());
             ys.remove(0);
 
-            Rectangle2D tableBBox = table.getBBox().get(table.getFirstPage());
+            Rectangle2D tableBBox = table.getBBox().get(page);
             List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredLines();
             xs.forEach(x -> {
                 Line2D line = new Line2D.Double(new Point2D.Double(x, tableBBox.getMaxY()), new Point2D.Double(x, tableBBox.getMinY()));

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/RectangleTransformations.java

@@ -3,8 +3,10 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 import static java.lang.String.format;
 
 import java.awt.geom.Area;
+import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -20,6 +22,8 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 
 import lombok.AllArgsConstructor;
 import lombok.NoArgsConstructor;
@@ -149,7 +153,27 @@ public class RectangleTransformations {
                 previousRectangle = currentRectangle;
             }
         }
-        return rectangleListsWithGaps.stream().map(RectangleTransformations::rectangle2DBBox).toList();
+        return rectangleListsWithGaps.stream()
+                .map(RectangleTransformations::rectangle2DBBox)
+                .toList();
+    }
+
+
+    public static CleanRulings extractRulings(List<? extends Rectangle2D.Float> rectangles) {
+
+        // Underlined or strikethrough are also in rulings but we dont want to split blocks with them so we use cells.
+        List<Ruling> horizontalRulings = new ArrayList<>();
+        List<Ruling> verticalRulings = new ArrayList<>();
+
+        rectangles.forEach(rectangle -> {
+            horizontalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y), new Point2D.Float(rectangle.x + rectangle.width, rectangle.y)));
+            horizontalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y + rectangle.height),
+                                             new Point2D.Float(rectangle.x + rectangle.width, rectangle.y + rectangle.height)));
+            verticalRulings.add(new Ruling(new Point2D.Float(rectangle.x, rectangle.y), new Point2D.Float(rectangle.x, rectangle.y + rectangle.height)));
+            verticalRulings.add(new Ruling(new Point2D.Float(rectangle.x + rectangle.width, rectangle.y),
+                                           new Point2D.Float(rectangle.x + rectangle.width, rectangle.y + rectangle.height)));
+        });
+        return CleanRulings.builder().vertical(verticalRulings).horizontal(horizontalRulings).build();
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java

@@ -28,7 +28,20 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
         prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
         LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
         LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
+        Arrays.stream(finishedEvent.message().split("\n"))
+                .forEach(log::info);
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testLayoutParserEndToEnd_RED_8747() {
+
+        prepareStorage("files/SinglePages/MergedEntities.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
+        Arrays.stream(finishedEvent.message().split("\n"))
+                .forEach(log::info);
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -751,6 +751,19 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
     }
 
+    @Test
+    public void testMergedEntities_Page26() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/MergedEntities.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 6, 6, 5, 0);
+
+    }
+
 
     @SneakyThrows
     private void toHtml(ClassificationDocument document, String filename) {

layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/service/ViewerDocumentService.java

@@ -17,6 +17,7 @@ import java.util.stream.Collectors;
 
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdfwriter.compress.CompressParameters;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -287,7 +288,7 @@ public class ViewerDocumentService {
      */
         Observation.createNotStarted("ViewerDocumentService", registry).contextualName("incremental-save").observe(() -> {
             try (var out = new FileOutputStream(outputFile)) {
-                pdDocument.saveIncremental(out);
+                pdDocument.save(out, CompressParameters.NO_COMPRESSION);
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }