TAAS-104: merge visually intersecting Paragraphs

* fix static import

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/TableData.java

@@ -13,9 +13,9 @@ public class TableData {
 
     @Schema(description = "A list of Objects containing information about all rows in this table.")
     List<RowData> rowData;
-    @Schema(description = "Numer of columns in this table.")
+    @Schema(description = "Number of columns in this table.")
     Integer numberOfCols;
-    @Schema(description = "Numer of rows in this table.")
+    @Schema(description = "Number of rows in this table.")
     Integer numberOfRows;
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingFinishedEvent.java

@@ -2,9 +2,19 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 import java.util.Map;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 
 @Builder
-public record LayoutParsingFinishedEvent(Map<String, String> identifier, long duration, int numberOfPages, String message) {
+@Schema(description = "Object containing information about the layout parsing.")
+public record LayoutParsingFinishedEvent(
+        @Schema(description = "General purpose identifier. It is returned exactly the same way it is inserted with the LayoutParsingRequest.")
+        Map<String, String> identifier,//
+        @Schema(description = "The duration of a single layout parsing in ms.")
+        long duration,//
+        @Schema(description = "The number of pages of the parsed document.")
+        int numberOfPages,//
+        @Schema(description = "A general message. It contains some information useful for a developer, like the paths where the files are stored. Not meant to be machine readable.")
+        String message) {
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -3,24 +3,42 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 import java.util.Map;
 import java.util.Optional;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 import lombok.NonNull;
 
 @Builder
+@Schema(description = "Object containing all storage paths the service needs to know.")
 public record LayoutParsingRequest(
+        @Schema(description = "Enum specifying the type of layout parsing to be performed.", allowableValues = "{RedactManager, DocuMine, TAAS}")//
         @NonNull LayoutParsingType layoutParsingType,
-        Map<String, String> identifier,
-        @NonNull String originFileStorageId,
-        Optional<String> tablesFileStorageId,
-        Optional<String> imagesFileStorageId,
 
-        @NonNull String structureFileStorageId,
-        String researchDocumentStorageId,
-        @NonNull String textBlockFileStorageId,
-        @NonNull String positionBlockFileStorageId,
-        @NonNull String pageFileStorageId,
-        @NonNull String simplifiedTextStorageId,
-        @NonNull String viewerDocumentStorageId,
+        @Schema(description = "General purpose identifiers. They are not changed by the service at all and are returned as is in the response queue.")//
+        Map<String, String> identifier,
+
+        @Schema(description = "Path to the original PDF file.")//
+        @NonNull String originFileStorageId,//
+        @Schema(description = "Optional Path to the table extraction file.")//
+        Optional<String> tablesFileStorageId,//
+        @Schema(description = "Optional Path to the image classification file.")//
+        Optional<String> imagesFileStorageId,//
+
+        @Schema(description = "Path where the Document Structure File will be stored.")//
+        @NonNull String structureFileStorageId,//
+        @Schema(description = "Path where the Research Data File will be stored.")//
+        String researchDocumentStorageId,//
+        @Schema(description = "Path where the Document Text File will be stored.")//
+        @NonNull String textBlockFileStorageId,//
+        @Schema(description = "Path where the Document Positions File will be stored.")//
+        @NonNull String positionBlockFileStorageId,//
+        @Schema(description = "Path where the Document Pages File will be stored.")//
+        @NonNull String pageFileStorageId,//
+        @Schema(description = "Path where the Simplified Text File will be stored.")//
+        @NonNull String simplifiedTextStorageId,//
+        @Schema(description = "Path where the Viewer Document PDF will be stored.")//
+        @NonNull String viewerDocumentStorageId,//
+        @Deprecated//
+        @Schema(description = "Path where the Section Grid will be stored.")//
         @NonNull String sectionGridStorageId) {
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -10,13 +10,13 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
@@ -50,6 +50,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.mapper.Docu
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -116,16 +117,45 @@ public class LayoutParsingPipeline {
                     .identifier(layoutParsingRequest.identifier())
                     .numberOfPages(numberOfPages)
                     .duration(System.currentTimeMillis() - start)
-                    .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
+                    .message(format("""
+                                    Layout parsing has finished in %.02f s.
+                                    identifiers: %s
+                                    %s
+                                    Files have been saved with Ids:
+                                    Structure: %s
+                                    Text: %s
+                                    Positions: %s
+                                    PageData: %s
+                                    Simplified Text: %s
+                                    Viewer Doc: %s""",
+                            ((float) (System.currentTimeMillis() - start)) / 1000,
+                            layoutParsingRequest.identifier(),
+                            buildSemanticNodeCountMessage(numberOfPages, documentGraph.buildSemanticNodeCounts()),
                             layoutParsingRequest.structureFileStorageId(),
                             layoutParsingRequest.textBlockFileStorageId(),
                             layoutParsingRequest.positionBlockFileStorageId(),
-                            layoutParsingRequest.pageFileStorageId()))
+                            layoutParsingRequest.pageFileStorageId(),
+                            layoutParsingRequest.simplifiedTextStorageId(),
+                            layoutParsingRequest.viewerDocumentStorageId()))
                     .build();
         }
     }
 
 
+    private String buildSemanticNodeCountMessage(int numberOfPages, Map<NodeType, Long> semanticNodeCounts) {
+
+        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
+                numberOfPages,
+                semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER));
+    }
+
+
     @SneakyThrows
     public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
                                               PDDocument originDocument,
@@ -166,12 +196,9 @@ public class LayoutParsingPipeline {
                     stripper.getMaxCharHeight());
 
             ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER ->
-                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case TAAS ->
-                        taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case DOCUMINE ->
-                        docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
             };
             classificationPage.setCleanRulings(cleanRulings);
             classificationPage.setRotation(rotation);
@@ -211,13 +238,15 @@ public class LayoutParsingPipeline {
     }
 
 
-    private Map<String, Rectangle2D> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
-        Map<String, Rectangle2D> markedContentBboxes = new HashMap<>();
+    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+
+        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
         markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
         markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
         return markedContentBboxes;
     }
 
+
     private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
 
         if (!classificationPage.isLandscape()) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/AbstractPageBlock.java

@@ -72,9 +72,30 @@ public abstract class AbstractPageBlock {
     }
 
 
-    public boolean intersectsY(AbstractPageBlock atc) {
+    public boolean intersectsY(AbstractPageBlock apb) {
 
-        return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY();
+        return this.minY <= apb.getMaxY() && this.maxY >= apb.getMinY();
     }
 
+
+    public boolean almostIntersects(AbstractPageBlock apb, float yThreshold, float xThreshold) {
+
+        return this.almostIntersectsX(apb, xThreshold) && this.almostIntersectsY(apb, yThreshold);
+    }
+
+
+    private boolean almostIntersectsY(AbstractPageBlock apb, float threshold) {
+
+        return this.minY - threshold <= apb.getMaxY() && this.maxY + threshold >= apb.getMinY();
+    }
+
+
+    private boolean almostIntersectsX(AbstractPageBlock apb, float threshold) {
+
+        return this.minX - threshold <= apb.getMaxX() && this.maxX + threshold >= apb.getMinX();
+    }
+
+
+    public abstract boolean isEmpty();
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java

@@ -42,6 +42,6 @@ public class ClassificationPage {
 
     CleanRulings cleanRulings;
 
-    private Map<String, Rectangle2D> markedContentBboxPerType = new HashMap<>();
+    private Map<String, List<Rectangle2D>> markedContentBboxPerType = new HashMap<>();
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationSection.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
@@ -29,4 +30,10 @@ public class ClassificationSection {
         return tables;
     }
 
+
+    public List<AbstractPageBlock> getNonEmptyPageBlocks() {
+
+        return pageBlocks.stream().filter(pageBlock -> !pageBlock.isEmpty()).collect(Collectors.toList());
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java

@@ -100,6 +100,11 @@ public class Document implements GenericSemanticNode {
     }
 
 
+    public Map<NodeType, Long> buildSemanticNodeCounts() {
+
+        return streamAllSubNodes().collect(Collectors.groupingBy(SemanticNode::getType, Collectors.counting()));
+    }
+
     @Override
     public String toString() {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java

@@ -76,16 +76,22 @@ public class TableCell implements GenericSemanticNode {
         }
 
         if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = buildTextBlock();
         }
         return textBlock;
     }
 
 
+    private TextBlock buildTextBlock() {
+
+        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+    }
+
+
     @Override
     public String toString() {
 
-        return treeId + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
+        return treeId + ": " + NodeType.TABLE_CELL + ": " + this.buildTextBlock().buildSummary();
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/TablePageBlock.java

@@ -42,6 +42,12 @@ public class TablePageBlock extends AbstractPageBlock {
     }
 
 
+    @Override
+    public boolean isEmpty() {
+
+        return getColCount() == 0 || getRowCount() == 0;
+    }
+
     public List<List<Cell>> getRows() {
 
         if (rows == null) {
@@ -304,6 +310,8 @@ public class TablePageBlock extends AbstractPageBlock {
     }
 
 
+
+
     public String getTextAsHtml() {
 
         StringBuilder sb = new StringBuilder();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java

@@ -365,4 +365,11 @@ public class TextPageBlock extends AbstractPageBlock {
 
     }
 
+
+    @Override
+    public boolean isEmpty() {
+
+        return sequences.isEmpty();
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/TaasBlockificationService.java

@@ -1,11 +1,16 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;
 
+import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;
+
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 
@@ -22,8 +27,12 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirA
 public class TaasBlockificationService {
 
     private static final float THRESHOLD = 1f;
-    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f;
+    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f; // multiplied with text height
+    private static final float INTERSECTS_Y_THRESHOLD =  2 * HEIGHT_PADDING; // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
     private static final int X_GAP_SPLIT_CONSTANT = 50;
+    public static final int X_ALIGNMENT_THRESHOLD = 1;
+    public static final int SMALL_Y_GAP_THRESHOLD = 5;
+    public static final int NEGATIVE_X_GAP_THRESHOLD = -5;
 
 
     /**
@@ -39,14 +48,28 @@ public class TaasBlockificationService {
     public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
         List<TextPageBlock> classificationTextBlocks = constructFineGranularTextPageBlocks(textPositions, horizontalRulingLines, verticalRulingLines);
-
-        classificationTextBlocks = mergeFineGranularTextPageBlocks(classificationTextBlocks);
+        classificationTextBlocks = mergeTextPageBlocksAligningX(classificationTextBlocks);
+        classificationTextBlocks = mergeIntersectingTextBlocksUntilConvergence(classificationTextBlocks);
 
         return new ClassificationPage(new ArrayList<>(classificationTextBlocks.stream().map(classificationTextBlock -> (AbstractPageBlock) classificationTextBlock).toList()));
     }
 
 
-    private List<TextPageBlock> mergeFineGranularTextPageBlocks(List<TextPageBlock> classificationTextBlocks) {
+    private List<TextPageBlock> mergeIntersectingTextBlocksUntilConvergence(List<TextPageBlock> classificationTextBlocks) {
+
+        int currentSize = classificationTextBlocks.size();
+        while (true) {
+            classificationTextBlocks = mergeTextPageBlocksAlmostIntersecting(classificationTextBlocks);
+            if (classificationTextBlocks.size() == currentSize) {
+                break;
+            }
+            currentSize = classificationTextBlocks.size();
+        }
+        return classificationTextBlocks;
+    }
+
+
+    private List<TextPageBlock> mergeTextPageBlocksAligningX(List<TextPageBlock> classificationTextBlocks) {
 
         if (classificationTextBlocks.isEmpty()) {
             return new ArrayList<>();
@@ -61,8 +84,8 @@ public class TaasBlockificationService {
                 previousTextBlock = currentTextBlock;
                 continue;
             }
-            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < 1;
-            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < 5;
+            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
+            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < SMALL_Y_GAP_THRESHOLD;
             if (alignsXRight && smallYGap) {
                 currentTextBlocksToMerge.add(currentTextBlock);
             } else {
@@ -76,6 +99,23 @@ public class TaasBlockificationService {
     }
 
 
+    private List<TextPageBlock> mergeTextPageBlocksAlmostIntersecting(List<TextPageBlock> textPageBlocks) {
+
+        Set<TextPageBlock> alreadyMerged = new HashSet<>();
+        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
+        for (TextPageBlock textPageBlock : textPageBlocks) {
+            if (alreadyMerged.contains(textPageBlock)) {
+                continue;
+            }
+            alreadyMerged.add(textPageBlock);
+            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
+                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
+                    .toList());
+        }
+        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
+    }
+
+
     private void assignOrientations(List<TextPageBlock> classificationTextBlocks) {
 
         Iterator<TextPageBlock> itty = classificationTextBlocks.iterator();
@@ -149,7 +189,7 @@ public class TaasBlockificationService {
             boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
             boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());
             boolean positiveXGapInline = prev != null && maxX + X_GAP_SPLIT_CONSTANT < word.getMinXDirAdj() && sameLine;
-            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < -5;
+            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < NEGATIVE_X_GAP_THRESHOLD;
             boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
             boolean splitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
@@ -164,7 +204,7 @@ public class TaasBlockificationService {
 
                 Orientation prevOrientation = null;
                 if (!classificationTextBlocks.isEmpty()) {
-                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - 1).getOrientation();
+                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - X_ALIGNMENT_THRESHOLD).getOrientation();
                 }
 
                 TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java

@@ -64,7 +64,7 @@ public class DocumentGraphFactory {
 
     private void addSections(ClassificationDocument document, Context context) {
 
-        document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getPageBlocks(), section.getImages(), context));
+        document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getNonEmptyPageBlocks(), section.getImages(), context));
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SearchTextWithTextPositionFactory.java

@@ -18,7 +18,7 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class SearchTextWithTextPositionFactory {
 
-    public final int HEIGHT_PADDING = 2;
+    public static final int HEIGHT_PADDING = 2;
     // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away.
     // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height.
     // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/ViewerDocumentService.java

@@ -51,7 +51,6 @@ public class ViewerDocumentService {
     @SneakyThrows
     public void createViewerDocument(PDDocument pdDocument, Document document, OutputStream outputStream, boolean layerVisibilityDefaultValue) {
 
-        log.info("Start Viewer Document Creation");
         LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
         // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
         // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
@@ -122,7 +121,6 @@ public class ViewerDocumentService {
         dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
 //        dictionariesToUpdate.add(pdDocument.getDocument().getTrailer());
         pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
-        log.info("Saved Viewer Document");
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/MarkedContentUtils.java

@@ -19,7 +19,7 @@ public class MarkedContentUtils {
     public static final String HEADER = "Header";
     public static final String FOOTER = "Footer";
 
-    public Rectangle2D getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {
+    public List<Rectangle2D> getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {
 
         if (markedContents == null) {
             return null;
@@ -40,16 +40,16 @@ public class MarkedContentUtils {
             return null;
         }
 
-        return RectangleTransformations.rectangle2DBBox(markedContentByYPosition.values().stream()
+        return markedContentByYPosition.values().stream()
                 .map(textPositions -> new TextPositionSequence(textPositions.stream()
                         .toList(), 0, true)
                         .getRectangle())
-                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList()));
+                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList());
     }
 
 
-    public boolean intersects(TextPageBlock textBlock, Map<String, Rectangle2D> markedContentBboxPerType, String type) {
-        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight());
+    public boolean intersects(TextPageBlock textBlock, Map<String, List<Rectangle2D>> markedContentBboxPerType, String type) {
+        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).stream().anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
     }
 
 }

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.queue;
 
+import java.util.Arrays;
+
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
@@ -45,12 +47,12 @@ public class MessageHandler {
         }
         LayoutParsingFinishedEvent layoutParsingFinishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
         sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent);
-        log.info("Layout parsing finished {} in {} ms", layoutParsingRequest.identifier(), layoutParsingFinishedEvent.duration());
     }
 
 
     public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
 
+        Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
         rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
     }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java

@@ -0,0 +1,34 @@
+package com.knecon.fforesight.service.layoutparser.server;
+
+import java.util.Arrays;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class LayoutparserEnd2EndTest extends AbstractTest {
+
+    @Autowired
+    private LayoutParsingPipeline layoutParsingPipeline;
+
+
+    @Test
+    @SneakyThrows
+    public void testLayoutParserEndToEnd() {
+
+        prepareStorage("files/bdr/btd_19_053_1905391.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.TAAS);
+        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
+        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
+    }
+
+}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ExtractMarkedContentTest.java

@@ -0,0 +1,196 @@
+package com.knecon.fforesight.service.layoutparser.server.graph;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import lombok.SneakyThrows;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.cos.COSArray;
+import org.apache.pdfbox.cos.COSBase;
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSNumber;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureElement;
+import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureNode;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.text.PDFMarkedContentExtractor;
+import org.apache.pdfbox.text.TextPosition;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;
+
+/**
+ * @author mkl
+ */
+@Disabled
+public class ExtractMarkedContentTest {
+    final static File RESULT_FOLDER = new File("target/test-outputs", "extract");
+
+    @BeforeEach
+    public void setUpBeforeClass() throws Exception {
+        RESULT_FOLDER.mkdirs();
+    }
+
+    /**
+     * <a href="https://stackoverflow.com/questions/54956720/how-to-replace-a-space-with-a-word-while-extract-the-data-from-pdf-using-pdfbox">
+     * How to replace a space with a word while extract the data from PDF using PDFBox
+     * </a>
+     * <br/>
+     * <a href="https://drive.google.com/open?id=10ZkdPlGWzMJeahwnQPzE6V7s09d1nvwq">
+     * test.pdf
+     * </a> as "testWPhromma.pdf"
+     * <p>
+     * This test shows how to, in principle, extract tagged text.
+     * </p>
+     */
+    @Test
+    @SneakyThrows
+    public void testExtractTestWPhromma() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "testWPhromma.pdf");
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+    }
+
+    /**
+     * <a href="https://stackoverflow.com/questions/59192443/get-tags-related-bboxs-even-though-there-is-no-attributes-a-in-document-cata">
+     * Get tag's related BBox's even though there is no attributes (/A in document catalog structure) related to Layout in PDFBox?
+     * </a>
+     * <br/>
+     * <a href="https://drive.google.com/file/d/1_-tuWuReaTvrDsqQwldTnPYrMHSpXIWp/view?usp=sharing">
+     * res_multipage.pdf
+     * </a>
+     * <p>
+     * This test shows how to, in principle, extract tagged text from this document.
+     * </p>
+     */
+    @Test
+    public void testExtractResMultipage() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "res_multipage.pdf");
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+    }
+
+    /**
+     * <a href="https://issues.apache.org/jira/browse/PDFBOX-5613">
+     * PDFBOX-5613 - uncorrent paragraph split
+     * </a>
+     * <br/>
+     * <a href="https://issues.apache.org/jira/secure/attachment/13058595/Daily%20Report.pdf">
+     * Daily Report.pdf
+     * </a>
+     * <p>
+     * Making use of the marked content, the contents of the file can better be extracted.
+     * </p>
+     */
+    @Test
+    public void testExtractDailyReport() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "Daily Report.pdf");
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+        }
+
+    /**
+     * @see #testExtractTestWPhromma()
+     */
+    void showStructure(PDStructureNode node, Map<PDPage, Map<Integer, PDMarkedContent>> markedContents) {
+        String structType = null;
+        PDPage page = null;
+        if (node instanceof PDStructureElement) {
+            PDStructureElement element = (PDStructureElement) node;
+            structType = element.getStructureType();
+            page = element.getPage();
+        }
+        Map<Integer, PDMarkedContent> theseMarkedContents = markedContents.get(page);
+        System.out.printf("<%s>\n", structType);
+        for (Object object : node.getKids()) {
+            if (object instanceof COSArray) {
+                for (COSBase base : (COSArray) object) {
+                    if (base instanceof COSDictionary) {
+                        showStructure(PDStructureNode.create((COSDictionary) base), markedContents);
+                    } else if (base instanceof COSNumber) {
+                        showContent(((COSNumber)base).intValue(), theseMarkedContents);
+                    } else {
+                        System.out.printf("?%s\n", base);
+                    }
+                }
+            } else if (object instanceof PDStructureNode) {
+                showStructure((PDStructureNode) object, markedContents);
+            } else if (object instanceof Integer) {
+                showContent((Integer)object, theseMarkedContents);
+            } else {
+                System.out.printf("?%s\n", object);
+            }
+
+        }
+        System.out.printf("</%s>\n", structType);
+    }
+
+    /**
+     * @see #showStructure(PDStructureNode, Map)
+     * @see #testExtractTestWPhromma()
+     */
+    void showContent(int mcid, Map<Integer, PDMarkedContent> theseMarkedContents) {
+        PDMarkedContent markedContent = theseMarkedContents != null ? theseMarkedContents.get(mcid) : null;
+        List<Object> contents = markedContent != null ? markedContent.getContents() : Collections.emptyList();
+        StringBuilder textContent =  new StringBuilder();
+        for (Object object : contents) {
+            if (object instanceof TextPosition) {
+                textContent.append(((TextPosition)object).getUnicode());
+            } else {
+                textContent.append("?" + object);
+            }
+        }
+        System.out.printf("%s\n", textContent);
+    }
+}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -25,8 +25,8 @@ public class ViewerDocumentTest extends BuildDocumentTest {
 
         LayoutGridService layoutGridService = new LayoutGridService();
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
-        String fileName = "files/new/VV-511309_OCR.pdf";
-        Document document = buildGraph(fileName, LayoutParsingType.DOCUMINE);
+        String fileName = "files/bdr/notMergedParagraphs.pdf";
+        Document document = buildGraph(fileName, LayoutParsingType.TAAS);
         String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
         try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getFile()); var out = new FileOutputStream(tmpFileName)) {
             viewerDocumentService.createViewerDocument(pdDocument, document, out, true);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -101,8 +101,13 @@ public abstract class AbstractTest {
     protected LayoutParsingRequest prepareStorage(InputStream fileInputStream) {
 
         storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileInputStream);
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+    }
+
+    protected LayoutParsingRequest buildDefaultLayoutParsingRequest(LayoutParsingType layoutParsingType) {
+
         return LayoutParsingRequest.builder()
-                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
+                .layoutParsingType(layoutParsingType)
                 .originFileStorageId(ORIGIN_FILE_ID)
                 .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                 .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
@@ -116,7 +121,6 @@ public abstract class AbstractTest {
                 .build();
     }
 
-
     @SneakyThrows
     protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile) {
 
@@ -135,19 +139,7 @@ public abstract class AbstractTest {
         storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
         storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);
 
-        return LayoutParsingRequest.builder()
-                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
-                .originFileStorageId(ORIGIN_FILE_ID)
-                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
-                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
-                .structureFileStorageId(STRUCTURE_FILE_ID)
-                .textBlockFileStorageId(TEXT_FILE_ID)
-                .positionBlockFileStorageId(POSITION_FILE_ID)
-                .pageFileStorageId(PAGES_FILE_ID)
-                .simplifiedTextStorageId(SIMPLIFIED_ID)
-                .sectionGridStorageId(SECTION_GRID_ID)
-                .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
-                .build();
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
     }