More refactoring

Reading Order
More
2024-02-20 14:04:28 +01:00 · 2024-02-20 11:56:43 +01:00 · 2024-02-16 14:51:26 +01:00 · 2024-02-16 14:50:31 +01:00 · 2024-02-16 14:08:59 +01:00 · 2024-02-16 13:48:03 +01:00
159 changed files with 4797 additions and 911 deletions
@@ -38,3 +38,7 @@ build/

 ### VS Code ###
 .vscode/
+gradlew.bat
+gradlew
+gradle.properties
+gradle/
@@ -10,7 +10,7 @@ deploy:
  script:
    - echo "Building with gradle version ${BUILDVERSION}"
    - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
  artifacts:
    reports:
@@ -18,4 +18,4 @@ deploy:
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
    - if: $CI_COMMIT_BRANCH =~ /^release/
-    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_TAG
@@ -1 +1,89 @@
+# PDF Layout Parser Micro-Service: layout-parser  

+## Introduction
+The layout-parser micro-service is a powerful tool designed to efficiently extract structured information from PDF documents. Written in Java and utilizing Spring Boot 3, Apache PDFBox, and RabbitMQ, this micro-service excels at parsing PDFs and organizing their content into a meaningful and coherent layout structure. Notably, the layout-parser micro-service distinguishes itself by relying solely on advanced algorithms, rather than machine learning techniques.
+
+### Key Steps in the PDF Layout Parsing Process:
+
+* **Text Position Extraction:**  
+The micro-service leverages Apache PDFBox to extract precise text positions for each individual character within the PDF document.
+
+* **Word Segmentation and Text Block Formation:**  
+Employing an array of diverse algorithms, the micro-service initially identifies and segments words, creating distinct text blocks.
+
+* **Text Block Classification:**  
+The segmented text blocks are then subjected to classification algorithms. These algorithms categorize the text blocks based on their content and visual properties, distinguishing between sections, subsections, headlines, paragraphs, images, tables, table cells, headers, and footers.
+
+* **Layout Coherence Establishment:**  
+The classified text blocks are subsequently orchestrated into a cohesive layout structure. This process involves arranging sections, subsections, paragraphs, images, and other elements in a logical and structured manner.
+
+* **Output Generation in Various Formats:**  
+Once the layout structure is established, the micro-service generates output in multiple formats. These formats are designed for seamless integration with downstream micro-services. The supported formats include JSON, XML, and others, ensuring flexibility in downstream data consumption.
+
+### Optional Enhancements:
+
+* **ML-Based Table Extraction:**  
+For enhanced results, users have the option to incorporate machine learning-based table extraction. This feature can be activated by providing ML-generated results as a JSON file, which are then integrated seamlessly into the layout structure.
+
+* **Image Classification using ML:**  
+Additionally, for more accurate image classification, users can optionally feed ML-generated image classification results into the micro-service. Similar to the table extraction option, the micro-service processes the pre-parsed results in JSON format, thus optimizing the accuracy of image content identification.
+
+In conclusion, the layout-parser micro-service is a versatile PDF layout parsing solution crafted entirely around advanced algorithms, without reliance on machine learning. It proficiently extracts text positions, segments content into meaningful blocks, classifies these blocks, arranges them coherently, and outputs structured data for downstream micro-services. Optional integration with ML-generated table extractions and image classifications further enhances its capabilities.
+
+
+
+
+
+
+## Installation
+
+### Prerequisites
+
+Before building and using the layout-parser micro-service, please ensure you have the following software and tools installed:
+
+Java Development Kit (JDK) 17 or later
+Gradle build tool (preinstalled)
+Build and Test
+To build and test the micro-service, follow these steps:
+
+### Clone the Repository:
+
+bash
+```
+git clone ssh://git@git.knecon.com:22222/fforesight/layout-parser.git
+cd layout-parser
+```
+### Build the Project:
+Use the following command to build the project using Gradle:
+
+```
+gradle clean build
+```
+### Run Tests:
+Run the test suite using the following command:
+```
+gradle test
+```
+## Building a Custom Docker Image
+To create a custom Docker image for the layout-parser micro-service, execute the provided script:
+
+### Ensure Docker is Installed:
+Ensure that Docker is installed and running on your system.
+
+### Run the Image Building Script:
+Execute the publish-custom-image script in the project directory:
+
+```
+./publish-custom-image
+```
+## Publishing to Internal Maven Repository
+To publish the layout-parser micro-service to your internal Maven repository, execute the following command:
+
+```
+gradle -Pversion=buildVersion publish
+```
+Replace buildVersion with the desired version number.
+
+## Additional Notes
+Make sure to configure any necessary application properties before deploying the micro-service.
+For advanced usage and configurations, refer to Kilian or Dom or preferably the source code.
@@ -24,6 +24,8 @@ tasks.named<Test>("test") {
    reports {
        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
    }
+    minHeapSize = "512m"
+    maxHeapSize = "2048m"
 }

 tasks.test {
@@ -9,12 +9,13 @@
    </description>

    <rule ref="category/java/errorprone.xml">
-        <exclude name="DataflowAnomalyAnalysis"/>
        <exclude name="MissingSerialVersionUID"/>
-        <exclude name="NullAssignment"/>
        <exclude name="AvoidLiteralsInIfCondition"/>
        <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AssignmentInOperand"/>
+        <exclude name="BeanMembersShouldSerialize"/>
    </rule>

-</ruleset>
+</ruleset>
+
@@ -10,13 +10,14 @@


    <rule ref="category/java/errorprone.xml">
-        <exclude name="DataflowAnomalyAnalysis"/>
        <exclude name="MissingSerialVersionUID"/>
-        <exclude name="NullAssignment"/>
        <exclude name="AvoidLiteralsInIfCondition"/>
        <exclude name="AvoidDuplicateLiterals"/>
        <exclude name="AvoidFieldNameMatchingMethodName"/>
-        <exclude name="AvoidFieldNameMatchingTypeName"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AssignmentInOperand"/>
+        <exclude name="TestClassWithoutTestCases"/>
+        <exclude name="BeanMembersShouldSerialize"/>
    </rule>

-</ruleset>
+</ruleset>
@@ -1,6 +1,10 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.1.0"
+    id("io.freefair.lombok") version "8.4"
 }

 description = "layoutparser-service-internal-api"
+
+dependencies {
+    implementation("io.swagger.core.v3:swagger-annotations:2.2.15")
+}
@@ -1,5 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -10,11 +13,16 @@ import lombok.experimental.FieldDefaults;
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class DocumentData {
+@Schema(description = "Object containing the complete document layout parsing information. It is split into 4 categories, structure, text, positions and pages: " + "The document tree structure of SemanticNodes such as Section, Paragraph, Headline, etc. " + "The text, which is stored as separate blocks of data. " + "The text positions, which are also stored as separate blocks. The Blocks are equal to the text blocks in length and order. " + "The page information.")
+public class DocumentData implements Serializable {

+    @Schema(description = "Contains information about the document's pages.")
    DocumentPage[] documentPages;
+    @Schema(description = "Contains information about the document's text.")
    DocumentTextData[] documentTextData;
+    @Schema(description = "Contains information about the document's text positions.")
    DocumentPositionData[] documentPositions;
+    @Schema(description = "Contains information about the document's semantic structure.")
    DocumentStructure documentStructure;

 }
@@ -1,5 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -12,11 +15,16 @@ import lombok.experimental.FieldDefaults;
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentPage {
+@Schema(description = "Object containing information about the document's pages.")
+public class DocumentPage implements Serializable {

+    @Schema(description = "The page number, starting with 1.")
    int number;
+    @Schema(description = "The page height in PDF user units.", example = "792")
    int height;
+    @Schema(description = "The page width in PDF user units.", example = "694")
    int width;
+    @Schema(description = "The page rotation as specified by the PDF.", example = "90", allowableValues = {"0", "90", "180", "270"})
    int rotation;

 }
@@ -1,5 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -12,10 +15,14 @@ import lombok.experimental.FieldDefaults;
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentPositionData {
+@Schema(description = "Object containing text positional information of a specific text block. A document is split into multiple text blocks, which are supposed to be read in order. Every text block can only occur on a single page.")
+public class DocumentPositionData implements Serializable {

+    @Schema(description = "Identifier of the text block.")
    Long id;
+    @Schema(description = "For each string coordinate in the search text of the text block, the array contains an entry relating the string coordinate to the position coordinate. This is required due to the text and position coordinates not being equal.")
    int[] stringIdxToPositionIdx;
+    @Schema(description = "The bounding box for each glyph as a rectangle. This matrix is of size (n,4), where n is the number of glyphs in the text block. The second dimension specifies the rectangle with the value x, y, width, height, with x, y specifying the lower left corner. In order to access this information, the stringIdxToPositionIdx array must be used to transform the coordinates.")
    float[][] positions;

 }
@@ -1,11 +1,13 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

 import java.awt.geom.Rectangle2D;
+import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -18,19 +20,22 @@ import lombok.experimental.FieldDefaults;
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentStructure {
+@Schema(description = "Object containing information about the parsed tree structure of the SemanticNodes, such as Section, Paragraph, Headline etc inside of the document.")
+public class DocumentStructure implements Serializable {

+    @Schema(description = "The root EntryData represents the Document.")
    EntryData root;

-
-    public static class TableProperties {
+    @Schema(description = "Object containing the extra field names, a table has in its properties field.")
+    public static class TableProperties implements Serializable {

        public static final String NUMBER_OF_ROWS = "numberOfRows";
        public static final String NUMBER_OF_COLS = "numberOfCols";

    }

-    public static class ImageProperties {
+    @Schema(description = "Object containing the extra field names, an Image has in its properties field.")
+    public static class ImageProperties implements Serializable {

        public static final String TRANSPARENT = "transparent";
        public static final String IMAGE_TYPE = "imageType";
@@ -39,7 +44,8 @@ public class DocumentStructure {

    }

-    public static class TableCellProperties {
+    @Schema(description = "Object containing the extra field names, a table cell has in its properties field.")
+    public static class TableCellProperties implements Serializable {

        public static final String B_BOX = "bBox";
        public static final String ROW = "row";
@@ -50,6 +56,7 @@ public class DocumentStructure {

    public static final String RECTANGLE_DELIMITER = ";";

+
    public static Rectangle2D parseRectangle2D(String bBox) {

        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
@@ -93,13 +100,20 @@ public class DocumentStructure {
    @NoArgsConstructor
    @AllArgsConstructor
    @FieldDefaults(level = AccessLevel.PRIVATE)
-    public static class EntryData {
+    @Schema(description = "Object containing information of a SemanticNode and also structuring the layout with children.")
+    public static class EntryData implements Serializable {

+        @Schema(description = "Type of the semantic node.", allowableValues = {"DOCUMENT", "SECTION", "PARAGRAPH", "HEADLINE", "TABLE", "TABLE_CELL", "HEADER", "FOOTER", "IMAGE"})
        NodeType type;
+        @Schema(description = "Specifies the position in the parsed tree structure.", example = "[1, 0, 2]")
        int[] treeId;
+        @Schema(description = "Specifies the text block IDs associated with this semantic node. The value should be joined with the DocumentTextData/DocumentPositionData. Is empty, if no text block is directly associated with this semantic node. Only Paragraph, Headline, Header or Footer is directly associated with a text block.", example = "[1]")
        Long[] atomicBlockIds;
+        @Schema(description = "Specifies the pages this semantic node appears on. The value should be joined with the PageData.", example = "[1, 2, 3]")
        Long[] pageNumbers;
+        @Schema(description = "Some semantic nodes have additional information, this information is stored in this Map. The extra fields are specified by the Properties subclasses.", example = "For a Table: {\"numberOfRows\": 3, \"numberOfCols\": 4}")
        Map<String, String> properties;
+        @Schema(description = "All child Entries of this Entry.", example = "[1, 2, 3]")
        List<EntryData> children;


@@ -1,7 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

+import java.io.Serializable;

-
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -14,14 +15,22 @@ import lombok.experimental.FieldDefaults;
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentTextData {
+@Schema(description = "Object containing text information of a specific text block. A document is split into multiple text blocks, which are supposed to be read in order. Every text block can only occur on a single page.")
+public class DocumentTextData implements Serializable {

+    @Schema(description = "Identifier of the text block.")
    Long id;
+    @Schema(description = "The page the text block occurs on.")
    Long page;
+    @Schema(description = "The text the text block.")
    String searchText;
+    @Schema(description = "Each text block is assigned a number on a page, starting from 0.")
    int numberOnPage;
+    @Schema(description = "The text blocks are ordered, this number represents the start of the text block as a string offset.")
    int start;
+    @Schema(description = "The text blocks are ordered, this number represents the end of the text block as a string offset.")
    int end;
+    @Schema(description = "The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.", example = "[5, 10]")
    int[] lineBreaks;

 }
@@ -1,8 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

+import java.io.Serializable;
 import java.util.Locale;

-public enum NodeType {
+public enum NodeType implements Serializable {
    DOCUMENT,
    SECTION,
    HEADLINE,
@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -9,9 +10,12 @@ import lombok.NoArgsConstructor;
@Builder
@NoArgsConstructor
@AllArgsConstructor
+@Schema(description = "Object containing a simplified version, which contains almost exclusively text, of the document structure Section class.")
 public class SimplifiedSectionText {

+    @Schema(description = "The number of this Section. This is used to map the simplified section text back to the original Section.")
    private int sectionNumber;
+    @Schema(description = "The text in this Section.")
    private String text;

 }
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 import java.util.ArrayList;
 import java.util.List;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -12,9 +13,12 @@ import lombok.NoArgsConstructor;
@Builder
@NoArgsConstructor
@AllArgsConstructor
+@Schema(description = "Object containing a simplified version, which contains almost exclusively text, of the document structure.")
 public class SimplifiedText {

+    @Schema(description = "Number of pages in the entire document.")
    private int numberOfPages;
+    @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();

 }
@@ -2,20 +2,29 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;

 import java.util.List;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 import lombok.Data;

@Data
@Builder
+@Schema(description = "Object containing information about a Paragraph/Headline/Header/Footer.")
 public class ParagraphData {

+    @Schema(description = "The text of this Semantic Node, without any linebreaks.", example = "This is some text.")
    private String text;
+    @Schema(description = "A list of text ranges in string offsets. Every character in any of the ranges is bold.", example = "[0, 15]")
    List<Range> boldTextBoundaries;
+    @Schema(description = "A list of text ranges in string offsets. Every character in any of the ranges is italic.", example = "[0, 15]")
    List<Range> italicTextBoundaries;
+    @Schema(description = "The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.", example = "[5, 10]")
    List<Integer> linebreaks;
+    @Schema(description = "The classification of this Paragraph.", allowableValues = "{paragraph, headline, header, footer}")
    private String classification;

+    @Schema(description = "Describes the text orientation of this semantic node. Any semantic node only has a single text orientation.", allowableValues = "{ZERO, QUARTER_CIRCLE, HALF_CIRCLE, THREE_QUARTER_CIRCLE}")
    private String orientation;
+    @Schema(description = "Describes the text direction in degrees of this semantic node. Any semantic node only has a single text direction.", minimum = "0", maximum = "359")
    private int textDirection;

 }
@@ -1,5 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;

+import io.swagger.v3.oas.annotations.media.Schema;
+
+@Schema(description = "Object specifying the start and end offsets of a text range in string offsets.")
 public record Range(int start, int end) {

 }
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;

 import java.util.List;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -9,8 +10,12 @@ import lombok.Data;
@Builder
@Data
@AllArgsConstructor
+@Schema(description = "Object containing a simplified version of the document structure. This simplified form only knows Paragraphs and Tables. The Paragraph Objects might be a Paragraph, Headline, Header or Footer.")
 public class ResearchDocumentData {

+    @Schema(description = "File name of the original uploaded file.")
    String originalFile;
+    @Schema(description = "A List of all paragraphs/headline or table objects, that have been parsed in this document.")
    List<StructureObject> structureObjects;
+
 }
@@ -2,14 +2,19 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;

 import java.util.List;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Data;

@Data
@AllArgsConstructor
+@Schema(description = "Object containing information about a Table Row.")
 public class RowData {

+    @Schema(description = "Boolean indicating whether this table row is classified as a header row.")
    boolean header;
+    @Schema(description = "A list of Objects containing information about the text in each cell of this row.")
    List<ParagraphData> cellText;
+    @Schema(description = "The bounding box of this StructureObject. Is always exactly 4 values representing x, y, w, h, where x, y specify the lower left corner.")
    float[] bBox;
 }
@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -7,13 +8,20 @@ import lombok.Data;
@Data
@Builder
@AllArgsConstructor
+@Schema(description = "Object containing information about either a Paragraph/Headline/Header/Footer or a Table.")
 public class StructureObject {

+    @Schema(description = "The ID of this StructureObject.")
    Integer structureObjectNumber;
+    @Schema(description = "This value indicates the start of the string offsets in this Object, with respect to the reading order.")
    int page;
+    @Schema(description = "This stringOffset indicates the start of the string offsets in this Object, with respect to the reading order of the entire document. It is equal to the previous' StructureObject stringOffset + its length.")
    int stringOffset;
+    @Schema(description = "The bounding box of this StructureObject. Is always exactly 4 values representing x, y, w, h, where x, y specify the lower left corner.", example = "[100, 100, 50, 50]")
    float[] boundingBox;
+    @Schema(description = "Object containing information about a Paragraph/Headline/Header/Footer. Either this or table is null.")
    ParagraphData paragraph;
+    @Schema(description = "Object containing information about a Table. Either this or paragraph is null.")
    TableData table;

 }
@@ -2,15 +2,20 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;

 import java.util.List;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Data;

@Data
@AllArgsConstructor
+@Schema(description = "Object containing information about a Table.")
 public class TableData {

+    @Schema(description = "A list of Objects containing information about all rows in this table.")
    List<RowData> rowData;
+    @Schema(description = "Number of columns in this table.")
    Integer numberOfCols;
+    @Schema(description = "Number of rows in this table.")
    Integer numberOfRows;

 }
@@ -2,9 +2,19 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;

 import java.util.Map;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;

@Builder
-public record LayoutParsingFinishedEvent(Map<String, String> identifier, long duration, int numberOfPages, String message) {
+@Schema(description = "Object containing information about the layout parsing.")
+public record LayoutParsingFinishedEvent(
+        @Schema(description = "General purpose identifier. It is returned exactly the same way it is inserted with the LayoutParsingRequest.")
+        Map<String, String> identifier,//
+        @Schema(description = "The duration of a single layout parsing in ms.")
+        long duration,//
+        @Schema(description = "The number of pages of the parsed document.")
+        int numberOfPages,//
+        @Schema(description = "A general message. It contains some information useful for a developer, like the paths where the files are stored. Not meant to be machine readable.")
+        String message) {

 }
@@ -3,22 +3,39 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 import java.util.Map;
 import java.util.Optional;

+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 import lombok.NonNull;

@Builder
+@Schema(description = "Object containing all storage paths the service needs to know.")
 public record LayoutParsingRequest(
+        @Schema(description = "Enum specifying the type of layout parsing to be performed.", allowableValues = "{RedactManager, DocuMine, TAAS}")//
        @NonNull LayoutParsingType layoutParsingType,
+
+        @Schema(description = "General purpose identifiers. They are not changed by the service at all and are returned as is in the response queue.")//
        Map<String, String> identifier,
-        @NonNull String originFileStorageId,
-        Optional<String> tablesFileStorageId,
-        Optional<String> imagesFileStorageId,
-        @NonNull String structureFileStorageId,
-        String researchDocumentStorageId,
-        @NonNull String textBlockFileStorageId,
-        @NonNull String positionBlockFileStorageId,
-        @NonNull String pageFileStorageId,
-        @NonNull String simplifiedTextStorageId,
-        @NonNull String sectionGridStorageId) {
+
+        @Schema(description = "Path to the original PDF file.")//
+        @NonNull String originFileStorageId,//
+        @Schema(description = "Optional Path to the table extraction file.")//
+        Optional<String> tablesFileStorageId,//
+        @Schema(description = "Optional Path to the image classification file.")//
+        Optional<String> imagesFileStorageId,//
+
+        @Schema(description = "Path where the Document Structure File will be stored.")//
+        @NonNull String structureFileStorageId,//
+        @Schema(description = "Path where the Research Data File will be stored.")//
+        String researchDocumentStorageId,//
+        @Schema(description = "Path where the Document Text File will be stored.")//
+        @NonNull String textBlockFileStorageId,//
+        @Schema(description = "Path where the Document Positions File will be stored.")//
+        @NonNull String positionBlockFileStorageId,//
+        @Schema(description = "Path where the Document Pages File will be stored.")//
+        @NonNull String pageFileStorageId,//
+        @Schema(description = "Path where the Simplified Text File will be stored.")//
+        @NonNull String simplifiedTextStorageId,//
+        @Schema(description = "Path where the Viewer Document PDF will be stored.")//
+        @NonNull String viewerDocumentStorageId) {

 }
@@ -1,26 +1,27 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.1.0"
+    id("io.freefair.lombok") version "8.4"
 }

 description = "layoutparser-service-processor"

 val jacksonVersion = "2.15.2"
-val pdfBoxVersion = "3.0.0-RC1"
+val pdfBoxVersion = "3.0.0"

 dependencies {
    implementation(project(":layoutparser-service-internal-api"))
+    implementation(project(":viewer-doc-processor"))

-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.124.0") {
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0") {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
-    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.29.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")

    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
 }
@@ -2,22 +2,32 @@ package com.knecon.fforesight.service.layoutparser.processor;

 import static java.lang.String.format;

+import java.awt.geom.Rectangle2D;
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;

+import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
@@ -30,7 +40,6 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
-import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
@@ -40,94 +49,184 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;

+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

+@SuppressWarnings("PMD.CloseResource")
@Slf4j
@Service
@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class LayoutParsingPipeline {

-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final CvTableParsingAdapter cvTableParsingAdapter;
-    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final SectionsBuilderService sectionsBuilderService;
-    private final SectionGridCreatorService sectionGridCreatorService;
-    private final TaasClassificationService taasClassificationService;
-    private final RedactManagerClassificationService redactManagerClassificationService;
-    private final DocuMineClassificationService docuMineClassificationService;
-    private final SimplifiedSectionTextService simplifiedSectionTextService;
-    private final BodyTextFrameService bodyTextFrameService;
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final TaasBlockificationService taasBlockificationService;
-    private final DocuMineBlockificationService docuMineBlockificationService;
-    private final RedactManagerBlockificationService redactManagerBlockificationService;
+    ImageServiceResponseAdapter imageServiceResponseAdapter;
+    CvTableParsingAdapter cvTableParsingAdapter;
+    LayoutParsingStorageService layoutParsingStorageService;
+    SectionsBuilderService sectionsBuilderService;
+    TaasClassificationService taasClassificationService;
+    RedactManagerClassificationService redactManagerClassificationService;
+    DocuMineClassificationService docuMineClassificationService;
+    SimplifiedSectionTextService simplifiedSectionTextService;
+    BodyTextFrameService bodyTextFrameService;
+    RulingCleaningService rulingCleaningService;
+    TableExtractionService tableExtractionService;
+    TaasBlockificationService taasBlockificationService;
+    DocuMineBlockificationService docuMineBlockificationService;
+    RedactManagerBlockificationService redactManagerBlockificationService;
+    LayoutGridService layoutGridService;
+    ObservationRegistry observationRegistry;
+    DocstrumSegmentationService docstrumSegmentationService;
+

    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {

        long start = System.currentTimeMillis();
+        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

-        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
-            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
-            }
+        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
+        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);

-            TableServiceResponse tableServiceResponse = new TableServiceResponse();
-            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
-            }
-
-            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
-            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-
-            int numberOfPages = originDocument.getNumberOfPages();
-
-            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
-            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
-            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
-                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
-                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
-            }
-
-            return LayoutParsingFinishedEvent.builder()
-                    .identifier(layoutParsingRequest.identifier())
-                    .numberOfPages(numberOfPages)
-                    .duration(System.currentTimeMillis() - start)
-                    .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
-                            layoutParsingRequest.structureFileStorageId(),
-                            layoutParsingRequest.textBlockFileStorageId(),
-                            layoutParsingRequest.positionBlockFileStorageId(),
-                            layoutParsingRequest.pageFileStorageId()))
-                    .build();
+        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
        }
+
+        TableServiceResponse tableServiceResponse = new TableServiceResponse();
+        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+        }
+
+        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+                originFile,
+                imageServiceResponse,
+                tableServiceResponse,
+                layoutParsingRequest.identifier().toString());
+
+        log.info("Building document graph for {}", layoutParsingRequest.identifier());
+
+        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
+
+        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
+
+        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false);
+
+        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
+
+        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
+
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+            log.info("Building research document data for {}", layoutParsingRequest.identifier());
+            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+        }
+
+        if (!viewerDocumentFile.equals(originFile)) {
+            viewerDocumentFile.delete();
+        }
+        originFile.delete();
+
+        return LayoutParsingFinishedEvent.builder()
+                .identifier(layoutParsingRequest.identifier())
+                .numberOfPages(documentGraph.getNumberOfPages())
+                .duration(System.currentTimeMillis() - start)
+                .message(format("""
+                                Layout parsing has finished in %.02f s.
+                                identifiers: %s
+                                %s
+                                Files have been saved with Ids:
+                                Structure: %s
+                                Text: %s
+                                Positions: %s
+                                PageData: %s
+                                Simplified Text: %s
+                                Viewer Doc: %s""",
+                        ((float) (System.currentTimeMillis() - start)) / 1000,
+                        layoutParsingRequest.identifier(),
+                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
+                        layoutParsingRequest.structureFileStorageId(),
+                        layoutParsingRequest.textBlockFileStorageId(),
+                        layoutParsingRequest.positionBlockFileStorageId(),
+                        layoutParsingRequest.pageFileStorageId(),
+                        layoutParsingRequest.simplifiedTextStorageId(),
+                        layoutParsingRequest.viewerDocumentStorageId()))
+                .build();
+
+    }
+
+
+    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
+
+        AtomicReference<Document> documentReference = new AtomicReference<>();
+
+        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
+            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
+        });
+
+        return documentReference.get();
+    }
+
+
+    private String buildSemanticNodeCountMessage(int numberOfPages, Map<NodeType, Long> semanticNodeCounts) {
+
+        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
+                numberOfPages,
+                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
    }


    @SneakyThrows
+    @Observed(name = "LayoutParsingPipeline", contextualName = "parse-layout")
    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
-                                PDDocument originDocument,
-                                ImageServiceResponse imageServiceResponse,
-                                TableServiceResponse tableServiceResponse) {
+                                              File originFile,
+                                              ImageServiceResponse imageServiceResponse,
+                                              TableServiceResponse tableServiceResponse,
+                                              String identifier) {

+        PDDocument originDocument = openDocument(originFile);
+        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);

        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

-        originDocument.setAllSecurityToBeRemoved(true);
        long pageCount = originDocument.getNumberOfPages();

        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+
+            if (pageNumber % 100 == 0) {
+                // re-open document every once in a while to save on RAM. This has no significant performance impact.
+                // This is due to PDFBox caching all images and some other stuff with Soft References. This dereferences them and forces the freeing of memory.
+                originDocument.close();
+                originDocument = openDocument(originFile);
+            }
+
+            if (pageNumber % 100 == 0 || pageNumber == pageCount || pageNumber == 1) {
+                log.info("Extracting text on Page {} for {}", pageNumber, identifier);
+            }
+
            classificationDocument.setPages(classificationPages);
            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
            PDPage pdPage = originDocument.getPage(pageNumber - 1);
@@ -146,16 +245,39 @@ public class LayoutParsingPipeline {
            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);

            PDRectangle cropbox = pdPage.getCropBox();
-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                    stripper.getRulings(),
-                    stripper.getMinCharWidth(),
-                    stripper.getMaxCharHeight());
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
+
+            // Docstrum
+            AtomicInteger num = new AtomicInteger(pageNumber);
+            var zones = docstrumSegmentationService.segmentPage(stripper.getTextPositionSequences());
+
+            List<AbstractPageBlock> pageBlocks = new ArrayList<>();
+            AtomicInteger numOnPage = new AtomicInteger(1);
+//            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+            zones.forEach(zone -> {
+
+                List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+                zone.getLines().forEach(line -> {
+                    line.getWords().forEach(word -> {
+                        textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), num.get()));
+                    });
+                });
+
+                var cps = redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
+
+                cps.getTextBlocks().forEach(cp -> {
+                    pageBlocks.add(redactManagerBlockificationService.buildTextBlock(((TextPageBlock) cp).getSequences(), numOnPage.getAndIncrement()));
+                });
+            });
+
+//            ClassificationPage classificationPage = switch (layoutParsingType) {
+//                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
+//                case TAAS -> taasBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
+//                case DOCUMINE -> docuMineBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
+//            };
+
+            ClassificationPage classificationPage = new ClassificationPage(pageBlocks);

-            ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            };
            classificationPage.setCleanRulings(cleanRulings);
            classificationPage.setRotation(rotation);
            classificationPage.setLandscape(isLandscape);
@@ -163,34 +285,78 @@ public class LayoutParsingPipeline {
            classificationPage.setPageWidth(cropbox.getWidth());
            classificationPage.setPageHeight(cropbox.getHeight());

+            // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
+            classificationPage.setMarkedContentBboxPerType(convertMarkedContents(stripper.getMarkedContents()));
+
            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
                classificationPage.setImages(pdfImages.get(pageNumber));
                imageServiceResponseAdapter.findOcr(classificationPage);
            }

-            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            tableExtractionService.extractTables(cleanRulings, classificationPage);
+
            buildPageStatistics(classificationPage);
            increaseDocumentStatistics(classificationPage, classificationDocument);

            classificationPages.add(classificationPage);
        }

+        originDocument.close();

+        log.info("Calculating BodyTextFrame for {}", identifier);
        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
-
+        log.info("Classify TextBlocks for {}", identifier);
        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }

-        sectionsBuilderService.buildSections(classificationDocument);
-        sectionsBuilderService.addImagesToSections(classificationDocument);
+        List<ClassificationSection> sections = new ArrayList<>();
+        for (var page : classificationPages) {
+            page.getTextBlocks().forEach(block -> {
+                block.setPage(page.getPageNumber());
+                var section = sectionsBuilderService.buildTextBlock(List.of(block), "a");
+                sections.add(section);
+            });
+        }
+        classificationDocument.setSections(sections);

+        log.info("Building Sections for {}", identifier);
+//        sectionsBuilderService.buildSections(classificationDocument);
+//        sectionsBuilderService.addImagesToSections(classificationDocument);
        return classificationDocument;
    }

+
+    private void addNumberOfPagesToTrace(int numberOfPages, long size) {
+
+        if (observationRegistry.getCurrentObservation() != null) {
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileSize", String.valueOf(size));
+        }
+    }
+
+
+    @SneakyThrows
+    private PDDocument openDocument(File originFile) {
+
+        PDDocument document = Loader.loadPDF(originFile);
+        document.setAllSecurityToBeRemoved(true);
+        return document;
+    }
+
+
+    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+
+        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
+        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
+        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
+        return markedContentBboxes;
+    }
+
+
    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {

        if (!classificationPage.isLandscape()) {
@@ -221,6 +387,4 @@ public class LayoutParsingPipeline {

    }

-
-
 }
@@ -1,10 +1,24 @@
 package com.knecon.fforesight.service.layoutparser.processor;

+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;

+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+
+import io.micrometer.observation.ObservationRegistry;
+
@Configuration
@ComponentScan
 public class LayoutParsingServiceProcessorConfiguration {

+
+    @Bean
+    @Autowired
+    public ViewerDocumentService viewerDocumentService(ObservationRegistry registry) {
+
+        return new ViewerDocumentService(registry);
+    }
+
 }
@@ -1,22 +1,20 @@
 package com.knecon.fforesight.service.layoutparser.processor;

 import java.io.File;
-import java.io.FileOutputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.util.Optional;

-import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
@@ -26,6 +24,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -38,17 +37,24 @@ public class LayoutParsingStorageService {
    private final StorageService storageService;
    private final ObjectMapper objectMapper;

+    @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
+    public File getOriginFile(String storageId) throws IOException {

-    public PDDocument getOriginFile(String storageId) throws IOException {
+        File tempFile = createTempFile("document", ".pdf");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return tempFile;
+    }

-        try (var originDocumentInputStream = getObject(storageId)) {
-            File tempFile = createTempFile("document", ".pdf");
-            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
-                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
-                originDocumentInputStream.close();
-            }
-            return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
+
+    @Observed(name = "LayoutParsingStorageService", contextualName = "get-viewer-doc-file")
+    public Optional<File> getViewerDocFile(String storageId) throws IOException {
+
+        if (!storageService.objectExists(TenantContext.getTenantId(), storageId)) {
+            return Optional.empty();
        }
+        File tempFile = createTempFile("viewerDocument", ".pdf");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return Optional.of(tempFile);
    }


@@ -74,6 +80,7 @@ public class LayoutParsingStorageService {
    }


+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
@@ -83,12 +90,6 @@ public class LayoutParsingStorageService {
    }


-    public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) {
-
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid);
-    }
-
-
    public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.researchDocumentStorageId(), researchDocumentData);
@@ -120,11 +121,13 @@ public class LayoutParsingStorageService {
    }


+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-simplified-text")
    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
    }

+
    @SneakyThrows
    private InputStream getObject(String storageId) {

@@ -134,4 +137,14 @@ public class LayoutParsingStorageService {
        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
    }

+
+    @SneakyThrows
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-viewer-document")
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, File out) {
+
+        try (var in = new FileInputStream(out)) {
+            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
+        }
+    }
+
 }
@@ -72,9 +72,30 @@ public abstract class AbstractPageBlock {
    }


-    public boolean intersectsY(AbstractPageBlock atc) {
+    public boolean intersectsY(AbstractPageBlock apb) {

-        return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY();
+        return this.minY <= apb.getMaxY() && this.maxY >= apb.getMinY();
    }

+
+    public boolean almostIntersects(AbstractPageBlock apb, float yThreshold, float xThreshold) {
+
+        return this.almostIntersectsX(apb, xThreshold) && this.almostIntersectsY(apb, yThreshold);
+    }
+
+
+    private boolean almostIntersectsY(AbstractPageBlock apb, float threshold) {
+
+        return this.minY - threshold <= apb.getMaxY() && this.maxY + threshold >= apb.getMinY();
+    }
+
+
+    private boolean almostIntersectsX(AbstractPageBlock apb, float threshold) {
+
+        return this.minX - threshold <= apb.getMaxX() && this.maxX + threshold >= apb.getMinX();
+    }
+
+
+    public abstract boolean isEmpty();
+
 }
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.ArrayList;
 import java.util.List;

-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText;

@@ -25,7 +24,6 @@ public class ClassificationDocument {
    private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
    private boolean headlines;

-    private SectionGrid sectionGrid = new SectionGrid();
    private long rulesVersion;

 }
@@ -1,7 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;

+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
@@ -9,6 +12,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRul
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;

 import lombok.Data;
+import lombok.Getter;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;

@@ -17,6 +21,7 @@ import lombok.RequiredArgsConstructor;
 public class ClassificationPage {

    @NonNull
+    @Getter
    private List<AbstractPageBlock> textBlocks;

    private List<ClassifiedImage> images = new ArrayList<>();
@@ -38,4 +43,6 @@ public class ClassificationPage {

    CleanRulings cleanRulings;

+    private Map<String, List<Rectangle2D>> markedContentBboxPerType = new HashMap<>();
+
 }
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;

 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
@@ -29,4 +30,10 @@ public class ClassificationSection {
        return tables;
    }

+
+    public List<AbstractPageBlock> getNonEmptyPageBlocks() {
+
+        return pageBlocks.stream().filter(pageBlock -> !pageBlock.isEmpty()).collect(Collectors.toList());
+    }
+
 }
@@ -12,6 +12,7 @@ import lombok.Setter;

@Setter
@EqualsAndHashCode
+@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
 public class Boundary implements Comparable<Boundary> {

    private int start;
@@ -10,7 +10,6 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

-import com.amazonaws.services.kms.model.NotFoundException;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -84,7 +83,7 @@ public class Document implements GenericSemanticNode {
    @Override
    public Headline getHeadline() {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build());
    }


@@ -100,6 +99,12 @@ public class Document implements GenericSemanticNode {
    }


+    public Map<NodeType, Long> buildSemanticNodeCounts() {
+
+        return streamAllSubNodes().collect(Collectors.groupingBy(SemanticNode::getType, Collectors.counting()));
+    }
+
+
    @Override
    public String toString() {

@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Footer implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -62,4 +67,14 @@ public class Footer implements GenericSemanticNode {
        return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Header implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public boolean isLeaf() {
@@ -62,4 +67,14 @@ public class Header implements GenericSemanticNode {
        return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,6 +36,9 @@ public class Headline implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -69,4 +74,14 @@ public class Headline implements GenericSemanticNode {
        return this;
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -32,6 +34,9 @@ public class Paragraph implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -60,4 +65,14 @@ public class Paragraph implements GenericSemanticNode {
        return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

+import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -35,6 +37,9 @@ public class Section implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;
+

    @Override
    public NodeType getType() {
@@ -74,4 +79,14 @@ public class Section implements GenericSemanticNode {
                .orElseGet(() -> getParent().getHeadline());
    }

+
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {
+
+        if (bBoxCache == null) {
+            bBoxCache = GenericSemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
+
 }
@@ -398,12 +398,11 @@ public interface SemanticNode {
     */
    default Map<Page, Rectangle2D> getBBox() {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        if (isLeaf()) {
-            return getBBoxFromLeafTextBlock(bBoxPerPage);
+            return getBBoxFromLeafTextBlock();
        }

-        return getBBoxFromChildren(bBoxPerPage);
+        return getBBoxFromChildren();
    }


@@ -426,25 +425,31 @@ public interface SemanticNode {

    /**
     * TODO: this produces unwanted results for sections spanning multiple columns.
-     *
-     * @param bBoxPerPage initial empty BoundingBox
+     * Computes the Union of the bounding boxes of all children recursively.
     * @return The union of the BoundingBoxes of all children
     */
-    private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {
+    private Map<Page, Rectangle2D> getBBoxFromChildren() {

-        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
-            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
-            return map2;
-        }).orElse(bBoxPerPage);
+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
+        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
+        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
+        for (Page page : pages) {
+            Rectangle2D bBoxOnPage = childrenBBoxes.stream()
+                    .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
+                    .map(childBboxPerPage -> childBboxPerPage.get(page))
+                    .collect(RectangleTransformations.collectBBox());
+            bBoxPerPage.put(page, bBoxOnPage);
+        }
+        return bBoxPerPage;
    }


    /**
-     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of all BoundingBoxes of the TextBlock of this node
     */
-    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
+    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {

+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
        return bBoxPerPage;
@@ -2,10 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

 import static java.lang.String.format;

+import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -34,13 +36,14 @@ public class Table implements SemanticNode {

    int numberOfRows;
    int numberOfCols;
-
    TextBlock textBlock;

    @Builder.Default
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

+    @EqualsAndHashCode.Exclude
+    Map<Page, Rectangle2D> bBoxCache;

    /**
     * Streams all entities in this table, that appear in a row, which contains any of the provided strings.
@@ -208,7 +211,6 @@ public class Table implements SemanticNode {
        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
    }

-
    /**
     * Streams all TableCells row-wise and filters them with header == true.
     *
@@ -313,5 +315,12 @@ public class Table implements SemanticNode {

        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
    }
+    @Override
+    public Map<Page, Rectangle2D> getBBox() {

+        if (bBoxCache == null) {
+            bBoxCache = SemanticNode.super.getBBox();
+        }
+        return bBoxCache;
+    }
 }
@@ -76,16 +76,22 @@ public class TableCell implements GenericSemanticNode {
        }

        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = buildTextBlock();
        }
        return textBlock;
    }


+    private TextBlock buildTextBlock() {
+
+        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+    }
+
+
    @Override
    public String toString() {

-        return treeId + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
+        return treeId + ": " + NodeType.TABLE_CELL + ": " + this.buildTextBlock().buildSummary();
    }

 }
@@ -1,12 +1,14 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -42,6 +44,12 @@ public class TablePageBlock extends AbstractPageBlock {
    }


+    @Override
+    public boolean isEmpty() {
+
+        return getColCount() == 0 || getRowCount() == 0;
+    }
+
    public List<List<Cell>> getRows() {

        if (rows == null) {
@@ -246,7 +254,8 @@ public class TablePageBlock extends AbstractPageBlock {
                if (prevY != null && prevX != null) {
                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));

-                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
+                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
+
                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
                    if (cell.hasMinimumSize()) {
                        row.add(cell);
@@ -267,6 +276,21 @@ public class TablePageBlock extends AbstractPageBlock {
    }


+
+    public boolean intersects(Cell cell1, Cell cell2) {
+        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
+            return false;
+        }
+        double x0 = cell1.getX() + 2;
+        double y0 = cell1.getY() + 2;
+        return (cell2.x + cell2.width > x0 &&
+                cell2.y + cell2.height > y0 &&
+                cell2.x < x0 + cell1.getWidth() -2 &&
+                cell2.y < y0 + cell1.getHeight() -2);
+    }
+
+
+
    @Override
    public String getText() {

@@ -304,6 +328,8 @@ public class TablePageBlock extends AbstractPageBlock {
    }


+
+
    public String getTextAsHtml() {

        StringBuilder sb = new StringBuilder();
@@ -17,7 +17,6 @@ import lombok.SneakyThrows;
@AllArgsConstructor
 public class RedTextPosition {

-    private String textMatrix;
    private float[] position;

    @JsonIgnore
@@ -46,6 +45,9 @@ public class RedTextPosition {
    @JsonIgnore
    private String fontName;

+    @JsonIgnore
+    private RedTextPosition parent;
+

    @SneakyThrows
    public static RedTextPosition fromTextPosition(TextPosition textPosition) {
@@ -56,8 +58,6 @@ public class RedTextPosition {

        pos.setFontSizeInPt(textPosition.getFontSizeInPt());

-        pos.setTextMatrix(textPosition.getTextMatrix().toString());
-
        var position = new float[4];

        position[0] = textPosition.getXDirAdj();
@@ -17,6 +17,7 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.Getter;
 import lombok.NoArgsConstructor;

@EqualsAndHashCode(callSuper = true)
@@ -27,6 +28,7 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {

    @Builder.Default
+    @Getter
    private List<TextPositionSequence> sequences = new ArrayList<>();

    @JsonIgnore
@@ -74,6 +76,7 @@ public class TextPageBlock extends AbstractPageBlock {
        return sequences.get(0).getPageWidth();
    }

+
    public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {

        List<TextPositionSequence> sequences = textBlocksToMerge.stream().map(TextPageBlock::getSequences).flatMap(java.util.Collection::stream).toList();
@@ -81,6 +84,7 @@ public class TextPageBlock extends AbstractPageBlock {
        return fromTextPositionSequences(sequences);
    }

+
    public static TextPageBlock fromTextPositionSequences(List<TextPositionSequence> wordBlockList) {

        TextPageBlock textBlock = null;
@@ -132,7 +136,6 @@ public class TextPageBlock extends AbstractPageBlock {
    }


-
    /**
     * Returns the minX value in pdf coordinate system.
     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -364,4 +367,11 @@ public class TextPageBlock extends AbstractPageBlock {

    }

+
+    @Override
+    public boolean isEmpty() {
+
+        return sequences.isEmpty();
+    }
+
 }
@@ -9,8 +9,6 @@ import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;

-import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;

@@ -57,6 +55,18 @@ public class TextPositionSequence implements CharSequence {
    }


+    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
+
+        this.textPositions = textPositions;
+        this.page = page;
+        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
+        this.rotation = textPositions.get(0).getRotation();
+        this.pageHeight = textPositions.get(0).getPageHeight();
+        this.pageWidth = textPositions.get(0).getPageWidth();
+        this.isParagraphStart = false;
+    }
+
+
    @Override
    public int length() {

@@ -142,8 +152,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minX value
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMinXDirAdj() {

        return textPositions.get(0).getXDirAdj();
@@ -157,8 +166,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxX value
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMaxXDirAdj() {

        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
@@ -172,8 +180,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMinYDirAdj() {

        return textPositions.get(0).getYDirAdj() - getTextHeight();
@@ -187,8 +194,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    public float getMaxYDirAdj() {

        return textPositions.get(0).getYDirAdj();
@@ -196,42 +202,38 @@ public class TextPositionSequence implements CharSequence {
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getHeight() {

        return getMaxYDirAdj() - getMinYDirAdj();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getWidth() {

        return getMaxXDirAdj() - getMinXDirAdj();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public String getFont() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "none";
+        }
        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public String getFontStyle() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "standard";
+        }
        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);

        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
@@ -243,20 +245,15 @@ public class TextPositionSequence implements CharSequence {
        } else {
            return "standard";
        }
-
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getFontSize() {

        return textPositions.get(0).getFontSizeInPt();
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
    public float getSpaceWidth() {

        return textPositions.get(0).getWidthOfSpace();
@@ -272,8 +269,7 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return bounding box of the word in Pdf Coordinate System
     */
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+
    @SneakyThrows
    public Rectangle getRectangle() {

@@ -8,6 +8,7 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.PageInfo;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;

@@ -24,24 +25,26 @@ public class CvTableParsingAdapter {
        Map<Integer, List<TableCells>> tableCells = new HashMap<>();
        tableServiceResponse.getData()
                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells())));
+                        .addAll(convertTableCells(tableData.getTableCells(), tableData.getPageInfo())));

        return tableCells;
    }


-    private Collection<TableCells> convertTableCells(List<TableCells> tableCells) {
+    private Collection<TableCells> convertTableCells(List<TableCells> tableCells, PageInfo pageInfo) {

        List<TableCells> cvParsedTableCells = new ArrayList<>();

-        tableCells.forEach(t -> cvParsedTableCells.add(TableCells.builder()
-                .y0(t.getY0())
-                .x1(t.getX1())
-                .y1(t.getY1())
-                .x0(t.getX0())
-                .width(t.getWidth())
-                .height(t.getHeight())
-                .build()));
+        tableCells.stream()
+                .filter(cell -> cell.getWidth() < pageInfo.getWidth() * 0.98 && cell.getHeight() < pageInfo.getHeight() * 0.98)
+                .forEach(t -> cvParsedTableCells.add(TableCells.builder()
+                        .y0(t.getY0())
+                        .x1(t.getX1())
+                        .y1(t.getY1())
+                        .x0(t.getX0())
+                        .width(t.getWidth())
+                        .height(t.getHeight())
+                        .build()));

        return cvParsedTableCells;
    }
@@ -1,5 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services;

+import java.util.Comparator;
 import java.util.List;

 import org.springframework.stereotype.Service;
@@ -12,73 +13,70 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

@Service
 public class BodyTextFrameService {

-
+    private static final float RULING_HEIGHT_THRESHOLD = 0.15f; // multiplied with page height. Header/Footer Rulings must be within that border of the page.
+    private static final float RULING_WIDTH_THRESHOLD = 0.75f; // multiplied with page width. Header/Footer Rulings must be at least that wide.

    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {

        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
        Rectangle landscapeBodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), true, layoutParsingType);
        for (ClassificationPage page : classificationDocument.getPages()) {
+//            var updatedBodyTextFrame = getBodyTextFrameFromRulings(page, bodyTextFrame, landscapeBodyTextFrame);
            setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
        }
    }


-/*
-    private Rectangle calculateBodyTextFrameByRulings(List<ClassificationPage> pages) {
+    private Rectangle getBodyTextFrameFromRulings(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {

-        Map<ClassificationPage, List<Ruling>> potentialHeaderRulingsPerPage = new HashMap<>();
-        Map<ClassificationPage, List<Ruling>> potentialFooterRulingsPerPage = new HashMap<>();
-
-        for (var page : pages) {
-            potentialHeaderRulingsPerPage.put(page,
-                    page.getCleanRulings()
-                            .getHorizontal()
-                            .stream()
-                            .filter(ruling -> ruling.getY1() > page.getPageHeight() * 0.8)
-                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
-                            .toList());
-            potentialFooterRulingsPerPage.put(page,
-                    page.getCleanRulings()
-                            .getHorizontal()
-                            .stream()
-                            .filter(ruling -> ruling.getY1() < page.getPageHeight() * 0.2)
-                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
-                            .toList());
+        List<Ruling> potentialFooterRulings = getPotentialFooterRulings(page);
+        List<Ruling> potentialHeaderRulings = getPotentialHeaderRulings(page);
+        var x = bodyTextFrame.getTopLeft().getX();
+        var y = bodyTextFrame.getTopLeft().getY();
+        var w = bodyTextFrame.getWidth();
+        var h = bodyTextFrame.getHeight();
+        if (!potentialFooterRulings.isEmpty()) {
+            h = y + h - potentialFooterRulings.get(0).getTop();
+            y = potentialFooterRulings.get(0).getTop();
        }
+        if (!potentialHeaderRulings.isEmpty()) {
+            h = potentialHeaderRulings.get(0).getBottom() - bodyTextFrame.getTopLeft().getY();
+        }
+        return new Rectangle(new Point(x, y), w, h, page.getPageNumber());
+    }

-        Optional<Ruling> headerRuling = potentialHeaderRulingsPerPage.values()
+
+    private List<Ruling> getPotentialFooterRulings(ClassificationPage page) {
+
+        return page.getCleanRulings()
+                .getHorizontal()
                .stream()
-                .flatMap(Collection::stream)
-                .filter(ruling -> potentialHeaderRulingsPerPage.values()
-                        .stream()
-                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
-                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
-                .min(Comparator.comparingDouble(Ruling::getY1));
+                .filter(ruling -> ruling.getY1() < page.getPageHeight() * RULING_HEIGHT_THRESHOLD)
+                .filter(ruling -> ruling.getWidth() > RULING_WIDTH_THRESHOLD * page.getPageWidth())
+                .sorted(Comparator.comparingDouble(Ruling::getTop))
+                .toList();
+    }

-        Optional<Ruling> footerRuling = potentialFooterRulingsPerPage.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .filter(ruling -> potentialHeaderRulingsPerPage.values()
-                        .stream()
-                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
-                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
-                .max(Comparator.comparingDouble(Ruling::getY1));

-        double maxY = headerRuling.isPresent() ? headerRuling.get().y1 : pages.stream().mapToDouble(ClassificationPage::getPageHeight).max().orElse(Double.MAX_VALUE);
-        double minY = footerRuling.map(ruling -> ruling.y1).orElse(0F);
-        double maxX = pages.stream().mapToDouble(ClassificationPage::getPageWidth).max().orElse(Double.MAX_VALUE);
+    private List<Ruling> getPotentialHeaderRulings(ClassificationPage page) {

-        return new Rectangle(new Point((float) maxX, (float) maxY), (float) 0, (float) minY, -1);
+        return page.getCleanRulings()
+                .getHorizontal()
+                .stream()
+                .filter(ruling -> ruling.getY1() > page.getPageHeight() * (1 - RULING_HEIGHT_THRESHOLD))
+                .filter(ruling -> ruling.getWidth() > RULING_WIDTH_THRESHOLD * page.getPageWidth())
+                .sorted(Comparator.comparingDouble(Ruling::getBottom).reversed())
+                .toList();
    }
-*/


    /**
@@ -129,10 +127,10 @@ public class BodyTextFrameService {
     * @param landscape               Calculate for landscape or portrait
     * @return Rectangle of the text frame
     */
-    private Rectangle calculateBodyTextFrame(List<ClassificationPage> pages,
-                                             FloatFrequencyCounter documentFontSizeCounter,
-                                             boolean landscape,
-                                             LayoutParsingType layoutParsingType) {
+    protected Rectangle calculateBodyTextFrame(List<ClassificationPage> pages,
+                                               FloatFrequencyCounter documentFontSizeCounter,
+                                               boolean landscape,
+                                               LayoutParsingType layoutParsingType) {

        float approximateHeaderLineCount;
        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
@@ -157,6 +155,11 @@ public class BodyTextFrameService {
                        continue;
                    }

+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                        continue;
+                    }
+
                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10) || !layoutParsingType.equals(
                            LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount) {
@@ -233,4 +236,4 @@ public class BodyTextFrameService {

    }

-}
+}
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services;

 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
@@ -26,35 +25,34 @@ public class PageContentExtractor {
    public List<PageContents> getSortedPageContents(String filename) throws IOException {

        List<PageContents> textPositionSequencesPerPage = new LinkedList<>();
-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
+        ClassPathResource pdfResource = new ClassPathResource(filename);

-            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile())) {

-                for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {

-                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-                    stripper.setPageNumber(pageNumber);
-                    stripper.setSortByPosition(true);
-                    stripper.setStartPage(pageNumber);
-                    stripper.setEndPage(pageNumber);
-                    stripper.setPdpage(pdPage);
-                    stripper.getText(pdDocument);
+                PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+                PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+                stripper.setPageNumber(pageNumber);
+                stripper.setSortByPosition(true);
+                stripper.setStartPage(pageNumber);
+                stripper.setEndPage(pageNumber);
+                stripper.setPdpage(pdPage);
+                stripper.getText(pdDocument);

-                    Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
-                            .stream()
-                            .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
+                Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
+                        .stream()
+                        .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));

-                    var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());
+                var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());

-                    textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
-                            RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
-                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox()),
-                            stripper.getRulings()));
-                }
+                textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
+                        RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
+                        RectangleTransformations.toRectangle2D(pdPage.getMediaBox()),
+                        stripper.getRulings()));
            }
        }
-
+    
        return textPositionSequencesPerPage;
    }

@@ -12,9 +12,9 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;

 import lombok.RequiredArgsConstructor;
@@ -25,10 +25,13 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RulingCleaningService {

-    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
+    private static final float THRESHOLD = 6;
+
+
+    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {

        if (!rulings.isEmpty()) {
-            snapPoints(rulings, minCharWidth, maxCharHeight);
+            snapPoints(rulings);
        }

        List<Ruling> vrs = new ArrayList<>();
@@ -53,14 +56,11 @@ public class RulingCleaningService {
        }
        List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);

-        return CleanRulings.builder()
-                .vertical(verticalRulingLines)
-                .horizontal(horizontalRulingLines)
-                .build();
+        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
    }


-    public void snapPoints(List<? extends Line2D.Float> rulings, float xThreshold, float yThreshold) {
+    public void snapPoints(List<? extends Line2D.Float> rulings) {

        // collect points and keep a Line -> p1,p2 map
        Map<Line2D.Float, Point2D[]> linesToPoints = new HashMap<>();
@@ -81,7 +81,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < xThreshold) {
+            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -108,7 +108,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < yThreshold) {
+            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -1,146 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BinaryOperator;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collector;
-import java.util.stream.Stream;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class SectionGridCreatorService {
-
-    public SectionGrid createSectionGrid(Document document) {
-
-        Map<Integer, List<SectionRectangle>> sectionBBox = document.streamAllSubNodesOfType(NodeType.SECTION).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> paragraphBBox = document.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> headlineBBox = document.streamAllSubNodesOfType(NodeType.HEADLINE).map(SemanticNode::getBBox).collect(new SectionGridCollector());
-        Map<Integer, List<SectionRectangle>> tableBBox = document.streamAllSubNodesOfType(NodeType.TABLE).map(node -> (Table) node).collect(new TableGridCollector());
-        var sectionGrid = new SectionGrid();
-
-        sectionGrid.setRectanglesPerPage(mergeMapsByConcatenatingLists(//
-                mergeMapsByConcatenatingLists(paragraphBBox, headlineBBox), //
-                mergeMapsByConcatenatingLists(sectionBBox, tableBBox)));
-
-        return sectionGrid;
-    }
-
-
-    private static abstract class GridCollector<T> implements Collector<T, Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> {
-
-        @Override
-        public Supplier<Map<Integer, List<SectionRectangle>>> supplier() {
-
-            return HashMap::new;
-        }
-
-
-        @Override
-        public Function<Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> finisher() {
-
-            return Function.identity();
-        }
-
-
-        @Override
-        public BinaryOperator<Map<Integer, List<SectionRectangle>>> combiner() {
-
-            return SectionGridCreatorService::mergeMapsByConcatenatingLists;
-        }
-
-
-        @Override
-        public Set<Characteristics> characteristics() {
-
-            return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
-        }
-
-    }
-
-    private static class TableGridCollector extends GridCollector<Table> {
-
-        @Override
-        public BiConsumer<Map<Integer, List<SectionRectangle>>, Table> accumulator() {
-
-            return (map, table) -> table.getPages()
-                    .forEach(page -> map.merge(page.getNumber(), List.of(toSectionRectangle(table, page, table.getPages().size())), SectionGridCreatorService::concatLists));
-        }
-
-
-        private static SectionRectangle toSectionRectangle(Table table, Page page, int numberOfParts) {
-
-            Rectangle2D rect = table.getBBox().get(page);
-            List<CellRectangle> tableCellRectangles = table.streamTableCells()
-                    .map(TableCell::getBBox)
-                    .map(map -> map.get(page))
-                    .filter(Objects::nonNull)
-                    .map(rectangle2D -> new CellRectangle(new Point((float) rectangle2D.getX(), (float) rectangle2D.getY()),
-                            (float) rectangle2D.getWidth(),
-                            (float) rectangle2D.getHeight()))
-                    .toList();
-            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()),
-                    (float) rect.getWidth(),
-                    (float) rect.getHeight(),
-                    1,
-                    numberOfParts,
-                    tableCellRectangles);
-        }
-
-    }
-
-    private static class SectionGridCollector extends GridCollector<Map<Page, Rectangle2D>> {
-
-        @Override
-        public BiConsumer<Map<Integer, List<SectionRectangle>>, Map<Page, Rectangle2D>> accumulator() {
-
-            return (mapToKeep, mapToMerge) -> mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page.getNumber(),
-                    List.of(toSectionRectangle(rectangle, mapToMerge.values().size())),
-                    SectionGridCreatorService::concatLists));
-
-        }
-
-
-        private static SectionRectangle toSectionRectangle(Rectangle2D rect, int numberOfParts) {
-
-            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()), (float) rect.getWidth(), (float) rect.getHeight(), 1, numberOfParts, null);
-        }
-
-    }
-
-
-    private static Map<Integer, List<SectionRectangle>> mergeMapsByConcatenatingLists(Map<Integer, List<SectionRectangle>> mapToKeep,
-                                                                                      Map<Integer, List<SectionRectangle>> mapToMerge) {
-
-        mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page, rectangle, SectionGridCreatorService::concatLists));
-        return mapToKeep;
-    }
-
-
-    private static List<SectionRectangle> concatLists(List<SectionRectangle> l1, List<SectionRectangle> l2) {
-
-        return Stream.concat(l1.stream(), l2.stream()).toList();
-    }
-
-}
@@ -240,7 +240,7 @@ public class SectionsBuilderService {
    }


-    private ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
+    public ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {

        ClassificationSection section = new ClassificationSection();

@@ -12,7 +12,6 @@ import java.util.Set;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -66,6 +65,17 @@ public class TableExtractionService {
    };


+    public boolean contains(Cell cell, double x, double y, double w, double h) {
+
+        if (cell.isEmpty() || w <= 0 || h <= 0) {
+            return false;
+        }
+        double x0 = cell.getX();
+        double y0 = cell.getY();
+        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
+    }
+
+
    /**
     * Finds tables on a page and moves textblocks into cells of the found tables.
     * Note: This algorithm uses Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -79,16 +89,17 @@ public class TableExtractionService {
     * @param cleanRulings The lines used to build the table.
     * @param page         Page object that contains textblocks and statistics.
     */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page, LayoutParsingType layoutParsingType) {
+    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {

-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical(), layoutParsingType);
+        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());

        List<TextPageBlock> toBeRemoved = new ArrayList<>();

        for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
            TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
            for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
+                if (cell.hasMinimumSize() && contains(cell,
+                        textBlock.getPdfMinX(),
                        textBlock.getPdfMinY(),
                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -102,7 +113,7 @@ public class TableExtractionService {
        cells = new ArrayList<>(new HashSet<>(cells));
        DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);

-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).toList();
+        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);

        List<TablePageBlock> tables = new ArrayList<>();
        for (Rectangle area : spreadsheetAreas) {
@@ -135,16 +146,14 @@ public class TableExtractionService {
    }


-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, LayoutParsingType layoutParsingType) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

-        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
-            // TODO: breaks some tables, for example "1 Abamectin Prr.pdf" try to fix this upstream in RulingCleaningService
-            for (Ruling r : horizontalRulingLines) {
-                if (r.getX2() < r.getX1()) {
-                    double a = r.getX2();
-                    r.x2 = (float) r.getX1();
-                    r.x1 = (float) a;
-                }
+        // Fix for 211.pdf
+        for (Ruling r : horizontalRulingLines) {
+            if (r.getX2() < r.getX1()) {
+                double a = r.getX2();
+                r.x2 = (float) r.getX1();
+                r.x1 = (float) a;
            }
        }

@@ -57,7 +57,7 @@ public class RedactManagerBlockificationService {
            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());

-            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
+            if (prev != null && (splitByDir || isSplitByRuling)) {

                Orientation prevOrientation = null;
                if (!chunkBlockList.isEmpty()) {
@@ -167,7 +167,7 @@ public class RedactManagerBlockificationService {
    }


-    private TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    public TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {

        TextPageBlock textBlock = null;

@@ -1,13 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;

-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;

-import org.springframework.stereotype.Service;
+// TODO: figure out, why this fails the build
+// import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -16,14 +11,25 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;

@Service
@SuppressWarnings("all")
 public class TaasBlockificationService {

    private static final float THRESHOLD = 1f;
-    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f;
+    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f; // multiplied with text height
+    private static final float INTERSECTS_Y_THRESHOLD = 4;// 2 * HEIGHT_PADDING // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
    private static final int X_GAP_SPLIT_CONSTANT = 50;
+    public static final int X_ALIGNMENT_THRESHOLD = 1;
+    public static final int NEGATIVE_X_GAP_THRESHOLD = -5;
+
+    private Pattern listIdentifier = Pattern.compile("^(?:(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)]))|\\uF0B7", Pattern.CASE_INSENSITIVE);


    /**
@@ -39,14 +45,28 @@ public class TaasBlockificationService {
    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

        List<TextPageBlock> classificationTextBlocks = constructFineGranularTextPageBlocks(textPositions, horizontalRulingLines, verticalRulingLines);
-
-        classificationTextBlocks = mergeFineGranularTextPageBlocks(classificationTextBlocks);
+        classificationTextBlocks = mergeTextPageBlocksAligningX(classificationTextBlocks);
+        classificationTextBlocks = mergeIntersectingTextBlocksUntilConvergence(classificationTextBlocks);

        return new ClassificationPage(new ArrayList<>(classificationTextBlocks.stream().map(classificationTextBlock -> (AbstractPageBlock) classificationTextBlock).toList()));
    }


-    private List<TextPageBlock> mergeFineGranularTextPageBlocks(List<TextPageBlock> classificationTextBlocks) {
+    private List<TextPageBlock> mergeIntersectingTextBlocksUntilConvergence(List<TextPageBlock> classificationTextBlocks) {
+
+        int currentSize = classificationTextBlocks.size();
+        while (true) {
+            classificationTextBlocks = mergeTextPageBlocksAlmostIntersecting(classificationTextBlocks);
+            if (classificationTextBlocks.size() == currentSize) {
+                break;
+            }
+            currentSize = classificationTextBlocks.size();
+        }
+        return classificationTextBlocks;
+    }
+
+
+    private List<TextPageBlock> mergeTextPageBlocksAligningX(List<TextPageBlock> classificationTextBlocks) {

        if (classificationTextBlocks.isEmpty()) {
            return new ArrayList<>();
@@ -55,16 +75,29 @@ public class TaasBlockificationService {
        List<TextPageBlock> currentTextBlocksToMerge = new LinkedList<>();
        textBlocksToMerge.add(currentTextBlocksToMerge);
        TextPageBlock previousTextBlock = null;
+        Float lastLineGap = null;
        for (TextPageBlock currentTextBlock : classificationTextBlocks) {
            if (previousTextBlock == null) {
                currentTextBlocksToMerge.add(currentTextBlock);
                previousTextBlock = currentTextBlock;
                continue;
            }
-            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < 1;
-            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < 5;
-            if (alignsXRight && smallYGap) {
+
+
+            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
+            boolean isListIdentifier = listIdentifierPattern.find();
+
+            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+
+            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
+//            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
+
+            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
+            boolean alignsXLeft = Math.abs(currentTextBlock.getPdfMinX() - previousTextBlock.getPdfMinX()) < X_ALIGNMENT_THRESHOLD;
+//            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < yGap;
+            if (yGap && sameFont && !isListIdentifier) {
                currentTextBlocksToMerge.add(currentTextBlock);
+
            } else {
                currentTextBlocksToMerge = new LinkedList<>();
                currentTextBlocksToMerge.add(currentTextBlock);
@@ -76,6 +109,23 @@ public class TaasBlockificationService {
    }


+    private List<TextPageBlock> mergeTextPageBlocksAlmostIntersecting(List<TextPageBlock> textPageBlocks) {
+
+        Set<TextPageBlock> alreadyMerged = new HashSet<>();
+        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
+        for (TextPageBlock textPageBlock : textPageBlocks) {
+            if (alreadyMerged.contains(textPageBlock)) {
+                continue;
+            }
+            alreadyMerged.add(textPageBlock);
+            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
+                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
+                    .toList());
+        }
+        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
+    }
+
+
    private void assignOrientations(List<TextPageBlock> classificationTextBlocks) {

        Iterator<TextPageBlock> itty = classificationTextBlocks.iterator();
@@ -128,8 +178,8 @@ public class TaasBlockificationService {


    private List<TextPageBlock> constructFineGranularTextPageBlocks(List<TextPositionSequence> textPositions,
-                                                                                        List<Ruling> horizontalRulingLines,
-                                                                                        List<Ruling> verticalRulingLines) {
+                                                                    List<Ruling> horizontalRulingLines,
+                                                                    List<Ruling> verticalRulingLines) {

        int indexOnPage = 0;
        List<TextPositionSequence> wordClusterToCombine = new ArrayList<>();
@@ -138,18 +188,18 @@ public class TaasBlockificationService {
        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
        TextPositionSequence prev = null;
        // TODO: make static final constant
-        var listIdentitifier = Pattern.compile("\\b(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)])", Pattern.CASE_INSENSITIVE);
+

        boolean wasSplitted = false;
        Float splitX1 = null;
        for (TextPositionSequence word : textPositions) {

-            Matcher listIdentifierPattern = listIdentitifier.matcher(word.toString());
+            Matcher listIdentifierPattern = listIdentifier.matcher(word.toString());

            boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
            boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());
            boolean positiveXGapInline = prev != null && maxX + X_GAP_SPLIT_CONSTANT < word.getMinXDirAdj() && sameLine;
-            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < -5;
+            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < NEGATIVE_X_GAP_THRESHOLD;
            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
            boolean splitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
@@ -164,7 +214,7 @@ public class TaasBlockificationService {

                Orientation prevOrientation = null;
                if (!classificationTextBlocks.isEmpty()) {
-                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - 1).getOrientation();
+                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - X_ALIGNMENT_THRESHOLD).getOrientation();
                }

                TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);
@@ -5,6 +5,7 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -62,12 +63,16 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
+        ) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
+        ) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -107,4 +112,4 @@ public class DocuMineClassificationService {
        }
    }

-}
+}
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -51,11 +52,13 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -56,9 +57,11 @@ public class TaasClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
            textBlock.setClassification(PageBlockType.HEADER);
-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
+                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -0,0 +1,153 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.LineBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.NearestNeighbourService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ReadingOrderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.SpacingService;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ZoneBuilderService;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class DocstrumSegmentationService {
+
+    private static final double MAX_VERTICAL_MERGE_DIST = 0.5;
+
+    private final NearestNeighbourService nearestNeighbourService;
+    private final SpacingService spacingService;
+    private final LineBuilderService lineBuilderService;
+    private final ZoneBuilderService zoneBuilderService;
+    private final ReadingOrderService readingOrderService;
+
+
+    public List<Zone> segmentPage(List<TextPositionSequence> textPositions) {
+
+        var positions = textPositions.stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
+
+        var characters = positions.stream().map(Character::new).collect(Collectors.toList());
+
+        nearestNeighbourService.findNearestNeighbors(characters);
+
+        var characterSpacing = spacingService.computeCharacterSpacing(characters);
+        var lineSpacing = spacingService.computeLineSpacing(characters);
+
+        var lines = lineBuilderService.buildLines(characters, characterSpacing, lineSpacing);
+
+        var zones = zoneBuilderService.buildZones(lines, characterSpacing, lineSpacing);
+
+        zones = mergeLinesInZones(zones, characterSpacing, Double.NEGATIVE_INFINITY, 0.0, 0.0, lineSpacing * MAX_VERTICAL_MERGE_DIST);
+
+        return readingOrderService.resolve(zones, false);
+    }
+
+//    private List<Zone> mergeZones(List<Zone> zones, double tolerance) {
+//
+//        List<BxBounds> bounds = new ArrayList<BxBounds>(zones.size());
+//        for (List<ComponentLine> zone : zones) {
+//            BxBoundsBuilder builder = new BxBoundsBuilder();
+//            for (ComponentLine line : zone) {
+//                for (Component component : line.getComponents()) {
+//                    builder.expand(component.getChunk().getBounds());
+//                }
+//            }
+//            bounds.add(builder.getBounds());
+//        }
+//
+//        List<List<ComponentLine>> outputZones = new ArrayList<List<ComponentLine>>();
+//        mainFor:
+//        for (int i = 0; i < zones.size(); i++) {
+//            for (int j = 0; j < zones.size(); j++) {
+//                if (i == j || bounds.get(j) == null || bounds.get(i) == null) {
+//                    continue;
+//                }
+//                if (BxModelUtils.contains(bounds.get(j), bounds.get(i), tolerance)) {
+//                    zones.get(j).addAll(zones.get(i));
+//                    bounds.set(i, null);
+//                    continue mainFor;
+//                }
+//            }
+//            outputZones.add(zones.get(i));
+//        }
+//        return outputZones;
+//    }
+
+
+    private List<Zone> mergeLinesInZones(List<Zone> zones,
+                                         double wordSpacing,
+                                         double minHorizontalDistance,
+                                         double maxHorizontalDistance,
+                                         double minVerticalDistance,
+                                         double maxVerticalDistance) {
+
+        List<Zone> outputZones = new ArrayList<>(zones.size());
+        for (Zone zone : zones) {
+            outputZones.add(mergeLinesInZone(zone, wordSpacing, minHorizontalDistance, maxHorizontalDistance, minVerticalDistance, maxVerticalDistance));
+        }
+        return outputZones;
+    }
+
+
+    private Zone mergeLinesInZone(Zone zone,
+                                  double wordSpacing,
+                                  double minHorizontalDistance,
+                                  double maxHorizontalDistance,
+                                  double minVerticalDistance,
+                                  double maxVerticalDistance) {
+
+        DisjointSets<Line> sets = new DisjointSets<>(zone.getLines());
+        for (int i = 0; i < zone.getLines().size(); i++) {
+            Line li = zone.getLines().get(i);
+            for (int j = i + 1; j < zone.getLines().size(); j++) {
+                Line lj = zone.getLines().get(j);
+                double hDist = li.horizontalDistance(lj);
+                double vDist = li.verticalDistance(lj);
+                if (minHorizontalDistance <= hDist && hDist <= maxHorizontalDistance && minVerticalDistance <= vDist && vDist <= maxVerticalDistance) {
+                    sets.union(li, lj);
+                } else if (minVerticalDistance <= vDist && vDist <= maxVerticalDistance && Math.abs(hDist - Math.min(li.getLength(), lj.getLength())) < 0.1) {
+                    boolean componentOverlap = false;
+                    int overlappingCount = 0;
+                    for (Character ci : li.getCharacters()) {
+                        for (Character cj : lj.getCharacters()) {
+                            double dist = ci.overlappingDistance(cj);
+                            if (dist > 2) {
+                                componentOverlap = true;
+                            }
+                            if (dist > 0) {
+                                overlappingCount++;
+                            }
+                        }
+                    }
+                    if (!componentOverlap && overlappingCount <= 2) {
+                        sets.union(li, lj);
+                    }
+                }
+            }
+        }
+        List<Line> outputZone = new ArrayList<>();
+        for (Set<Line> group : sets) {
+            List<Character> components = new ArrayList<>();
+            for (Line line : group) {
+                components.addAll(line.getCharacters());
+            }
+            components.sort(Comparator.comparingDouble(Character::getX));
+
+            outputZone.add(new Line(components, wordSpacing));
+        }
+        return new Zone(outputZone);
+    }
+
+}
@@ -0,0 +1,32 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+public class AngleFilter {
+
+    protected double lowerAngle;
+    protected double upperAngle;
+
+
+    public AngleFilter(double lowerAngle, double upperAngle) {
+
+        if (lowerAngle < -Math.PI / 2) {
+            lowerAngle += Math.PI;
+        }
+        if (upperAngle >= Math.PI / 2) {
+            upperAngle -= Math.PI;
+        }
+
+        this.lowerAngle = lowerAngle;
+        this.upperAngle = upperAngle;
+    }
+
+
+    public boolean matches(Neighbor neighbor) {
+
+        if (lowerAngle <= upperAngle) {
+            return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
+        } else {
+            return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
+        }
+    }
+
+}
@@ -0,0 +1,48 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+
+import lombok.Data;
+
+@Data
+public abstract class BoundingBox {
+
+    private Rectangle2D bBox;
+
+
+    public double getX() {
+
+        return bBox.getX();
+    }
+
+
+    public double getY() {
+
+        return bBox.getY();
+    }
+
+
+    public double getWidth() {
+
+        return bBox.getWidth();
+    }
+
+
+    public double getHeight() {
+
+        return bBox.getHeight();
+    }
+
+
+    public double getArea() {
+
+        return (bBox.getHeight() * bBox.getWidth());
+    }
+
+
+    public boolean contains(Rectangle2D contained, double tolerance) {
+
+        return bBox.getX() <= contained.getX() + tolerance && bBox.getY() <= contained.getY() + tolerance && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
+    }
+
+}
@@ -0,0 +1,84 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+
+import lombok.Data;
+
+@Data
+public class Character {
+
+    private final double x;
+    private final double y;
+    private final RedTextPosition textPosition;
+
+    private List<Neighbor> neighbors = new ArrayList<>();
+
+
+    public Character(RedTextPosition chunk) {
+
+        this.x = chunk.getXDirAdj() + chunk.getWidthDirAdj() / 2;
+        this.y = chunk.getYDirAdj() + chunk.getHeightDir() / 2;
+        this.textPosition = chunk;
+    }
+
+
+    public double getHeight() {
+
+        return textPosition.getHeightDir();
+    }
+
+
+    public double distance(Character character) {
+
+        double dx = getX() - character.getX();
+        double dy = getY() - character.getY();
+        return Math.sqrt(dx * dx + dy * dy);
+    }
+
+
+    public double horizontalDistance(Character character) {
+
+        return Math.abs(getX() - character.getX());
+    }
+
+
+    public double verticalDistance(Character character) {
+
+        return Math.abs(getY() - character.getY());
+    }
+
+
+    public double overlappingDistance(Character other) {
+
+        double[] xs = new double[4];
+        double s = Math.sin(-0), c = Math.cos(-0);
+        xs[0] = c * x - s * y;
+        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
+        xs[2] = c * other.x - s * other.y;
+        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
+        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
+        Arrays.sort(xs);
+        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
+    }
+
+
+    public void setNeighbors(List<Neighbor> neighbors) {
+
+        this.neighbors = neighbors;
+    }
+
+
+    public double angle(Character character) {
+
+        if (getX() > character.getX()) {
+            return Math.atan2(getY() - character.getY(), getX() - character.getX());
+        } else {
+            return Math.atan2(character.getY() - getY(), character.getX() - getX());
+        }
+    }
+
+}
@@ -0,0 +1,194 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.util.AbstractSet;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+public class DisjointSets<E> implements Iterable<Set<E>> {
+
+    private final Map<E, Entry<E>> map = new HashMap<>();
+
+
+    public DisjointSets(Collection<? extends E> collection) {
+
+        for (E element : collection) {
+            map.put(element, new Entry<E>(element));
+        }
+    }
+
+
+    public boolean areTogether(E e1, E e2) {
+
+        return map.get(e1).findRepresentative() == map.get(e2).findRepresentative();
+    }
+
+
+    public void union(E e1, E e2) {
+
+        Entry<E> r1 = map.get(e1).findRepresentative();
+        Entry<E> r2 = map.get(e2).findRepresentative();
+        if (r1 != r2) {
+            if (r1.size <= r2.size) {
+                r2.mergeWith(r1);
+            } else {
+                r1.mergeWith(r2);
+            }
+        }
+    }
+
+
+    @Override
+    public Iterator<Set<E>> iterator() {
+
+        return new Iterator<>() {
+
+            private final Iterator<Entry<E>> iterator = map.values().iterator();
+            private Entry<E> nextRepresentative;
+
+            {
+                findNextRepresentative();
+            }
+
+            @Override
+            public boolean hasNext() {
+
+                return nextRepresentative != null;
+            }
+
+
+            @Override
+            public Set<E> next() {
+
+                if (nextRepresentative == null) {
+                    throw new NoSuchElementException();
+                }
+                Set<E> result = nextRepresentative.asSet();
+                findNextRepresentative();
+                return result;
+            }
+
+
+            private void findNextRepresentative() {
+
+                while (iterator.hasNext()) {
+                    Entry<E> candidate = iterator.next();
+                    if (candidate.isRepresentative()) {
+                        nextRepresentative = candidate;
+                        return;
+                    }
+                }
+                nextRepresentative = null;
+            }
+
+
+            @Override
+            public void remove() {
+
+                throw new UnsupportedOperationException();
+            }
+
+        };
+    }
+
+
+    private static class Entry<E> {
+
+        private int size = 1;
+        private final E value;
+        private Entry<E> parent = this;
+        private Entry<E> next = null;
+        private Entry<E> last = this;
+
+
+        Entry(E value) {
+
+            this.value = value;
+        }
+
+
+        void mergeWith(Entry<E> otherRepresentative) {
+
+            size += otherRepresentative.size;
+            last.next = otherRepresentative;
+            last = otherRepresentative.last;
+            otherRepresentative.parent = this;
+        }
+
+
+        Entry<E> findRepresentative() {
+
+            Entry<E> representative = parent;
+            while (representative.parent != representative) {
+                representative = representative.parent;
+            }
+            for (Entry<E> entry = this; entry != representative; ) {
+                Entry<E> nextEntry = entry.parent;
+                entry.parent = representative;
+                entry = nextEntry;
+            }
+            return representative;
+        }
+
+
+        boolean isRepresentative() {
+
+            return parent == this;
+        }
+
+
+        Set<E> asSet() {
+
+            return new AbstractSet<E>() {
+
+                @Override
+                public Iterator<E> iterator() {
+
+                    return new Iterator<E>() {
+
+                        private Entry<E> nextEntry = findRepresentative();
+
+
+                        @Override
+                        public boolean hasNext() {
+
+                            return nextEntry != null;
+                        }
+
+
+                        @Override
+                        public E next() {
+
+                            if (nextEntry == null) {
+                                throw new NoSuchElementException();
+                            }
+                            E result = nextEntry.value;
+                            nextEntry = nextEntry.next;
+                            return result;
+                        }
+
+
+                        @Override
+                        public void remove() {
+
+                            throw new UnsupportedOperationException();
+                        }
+
+                    };
+                }
+
+
+                @Override
+                public int size() {
+
+                    return findRepresentative().size;
+                }
+            };
+        }
+
+    }
+
+}
@@ -0,0 +1,91 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+public class Histogram {
+
+    private static final double EPSILON = 1.0e-6;
+    private final double min;
+    private final double resolution;
+    private double[] frequencies;
+
+
+    public Histogram(double minValue, double maxValue, double resolution) {
+
+        this.min = minValue - EPSILON;
+        double delta = maxValue - minValue + 2 * EPSILON;
+        int size = Math.max(1, (int) Math.round((maxValue - minValue) / resolution));
+        this.resolution = delta / size;
+        this.frequencies = new double[size];
+    }
+
+
+    public void kernelSmooth(double[] kernel) {
+
+        double[] newFrequencies = new double[frequencies.length];
+        int shift = (kernel.length - 1) / 2;
+        for (int i = 0; i < kernel.length; i++) {
+            int jStart = Math.max(0, i - shift);
+            int jEnd = Math.min(frequencies.length, frequencies.length + i - shift);
+            for (int j = jStart; j < jEnd; j++) {
+                newFrequencies[j - i + shift] += kernel[i] * frequencies[j];
+            }
+        }
+        frequencies = newFrequencies;
+    }
+
+
+    public double[] createGaussianKernel(double length, double stdDeviation) {
+
+        int r = (int) Math.round(length / resolution) / 2;
+        stdDeviation /= resolution;
+
+        int size = 2 * r + 1;
+        double[] kernel = new double[size];
+        double sum = 0;
+        double b = 2 * stdDeviation * stdDeviation;
+        double a = 1 / Math.sqrt(Math.PI * b);
+        for (int i = 0; i < size; i++) {
+            kernel[i] = a * Math.exp(-(i - r) * (i - r) / b);
+            sum += kernel[i];
+        }
+        for (int i = 0; i < size; i++) {
+            kernel[i] /= sum;
+        }
+        return kernel;
+    }
+
+
+    public void gaussianSmooth(double windowLength, double stdDeviation) {
+
+        kernelSmooth(createGaussianKernel(windowLength, stdDeviation));
+    }
+
+
+    public void add(double value) {
+
+        frequencies[(int) ((value - min) / resolution)] += 1.0;
+    }
+
+
+    public int getSize() {
+
+        return frequencies.length;
+    }
+
+
+    public double getPeakValue() {
+
+        int peakIndex = 0;
+        for (int i = 1; i < frequencies.length; i++) {
+            if (frequencies[i] > frequencies[peakIndex]) {
+                peakIndex = i;
+            }
+        }
+        int peakEndIndex = peakIndex + 1;
+        final double EPS = 0.0001;
+        while (peakEndIndex < frequencies.length && Math.abs(frequencies[peakEndIndex] - frequencies[peakIndex]) < EPS) {
+            peakEndIndex++;
+        }
+        return ((double) peakIndex + peakEndIndex) / 2 * resolution + min;
+    }
+
+}
@@ -0,0 +1,165 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.Data;
+
+@Data
+public class Line extends BoundingBox {
+
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.2;
+
+    private final double x0;
+    private final double y0;
+
+    private final double x1;
+    private final double y1;
+
+    private final double height;
+
+    private final List<Character> characters;
+    private final List<TextPositionSequence> words = new ArrayList<>();
+
+
+    public Line(List<Character> characters, double wordSpacing) {
+
+        this.characters = characters;
+
+        if (characters.size() >= 2) {
+            // linear regression
+            double sx = 0.0, sxx = 0.0, sxy = 0.0, sy = 0.0;
+            for (Character character : characters) {
+                sx += character.getX();
+                sxx += character.getX() * character.getX();
+                sxy += character.getX() * character.getY();
+                sy += character.getY();
+            }
+            double b = (characters.size() * sxy - sx * sy) / (characters.size() * sxx - sx * sx);
+            double a = (sy - b * sx) / characters.size();
+
+            this.x0 = characters.get(0).getX();
+            this.y0 = a + b * this.x0;
+            this.x1 = characters.get(characters.size() - 1).getX();
+            this.y1 = a + b * this.x1;
+        } else {
+            Character character = characters.get(0);
+            double dx = character.getTextPosition().getWidthDirAdj() / 3;
+            double dy = dx * Math.tan(0);
+            this.x0 = character.getX() - dx;
+            this.x1 = character.getX() + dx;
+            this.y0 = character.getY() - dy;
+            this.y1 = character.getY() + dy;
+        }
+        height = computeHeight();
+        computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
+        buildBBox();
+    }
+
+
+    public double getAngle() {
+
+        return Math.atan2(y1 - y0, x1 - x0);
+    }
+
+
+    public double getLength() {
+
+        return Math.sqrt((x0 - x1) * (x0 - x1) + (y0 - y1) * (y0 - y1));
+    }
+
+
+    private double computeHeight() {
+
+        double sum = 0.0;
+        for (Character component : characters) {
+            sum += component.getHeight();
+        }
+        return sum / characters.size();
+    }
+
+
+    public double angularDifference(Line j) {
+
+        double diff = Math.abs(getAngle() - j.getAngle());
+        if (diff <= Math.PI / 2) {
+            return diff;
+        } else {
+            return Math.PI - diff;
+        }
+    }
+
+
+    public double horizontalDistance(Line other) {
+
+        double[] xs = new double[4];
+        xs[0] = x0;
+        xs[1] = x1;
+        xs[2] = other.x0;
+        xs[3] = other.x1;
+        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
+        Arrays.sort(xs);
+        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
+    }
+
+
+    public double verticalDistance(Line other) {
+
+        double ym = (y0 + y1) / 2;
+        double yn = (other.y0 + other.y1) / 2;
+        return Math.abs(ym - yn) / Math.sqrt(1);
+    }
+
+
+    private void computeWords(double wordSpacing) {
+
+        TextPositionSequence word = new TextPositionSequence();
+        Character previous = null;
+        for (Character current : characters) {
+            if (previous != null) {
+                double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
+                if (dist > wordSpacing) {
+                    words.add(word);
+                    word = new TextPositionSequence();
+                }
+            }
+            word.getTextPositions().add(current.getTextPosition());
+            previous = current;
+        }
+        words.add(word);
+    }
+
+
+    private void buildBBox() {
+
+        double minX = Double.POSITIVE_INFINITY;
+        double minY = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+        double maxY = Double.NEGATIVE_INFINITY;
+
+        for (Character character : characters) {
+
+            minX = Math.min(minX, character.getTextPosition().getXDirAdj());
+            minY = Math.min(minY, character.getTextPosition().getYDirAdj());
+            maxX = Math.max(maxX, character.getTextPosition().getXDirAdj() + character.getTextPosition().getWidthDirAdj());
+            maxY = Math.max(maxY, character.getTextPosition().getYDirAdj() + character.getTextPosition().getHeightDir());
+
+        }
+
+        this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
+    }
+
+
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        words.forEach(word -> sb.append(word.toString()).append(" "));
+        return sb.toString().trim();
+    }
+
+}
+
@@ -0,0 +1,36 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import lombok.Getter;
+
+public class Neighbor {
+
+    @Getter
+    private final double distance;
+    @Getter
+    private final double angle;
+    private final Character originCharacter;
+    @Getter
+    private final Character character;
+
+
+    public Neighbor(Character neighbor, Character origin) {
+
+        this.distance = neighbor.distance(origin);
+        this.angle = neighbor.angle(origin);
+        this.character = neighbor;
+        this.originCharacter = origin;
+    }
+
+
+    public double getHorizontalDistance() {
+
+        return character.horizontalDistance(originCharacter);
+    }
+
+
+    public double getVerticalDistance() {
+
+        return character.verticalDistance(originCharacter);
+    }
+
+}
@@ -0,0 +1,50 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Comparator;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class Zone extends BoundingBox {
+
+    private List<Line> lines;
+
+
+    public Zone(List<Line> lines) {
+
+        lines.sort(Comparator.comparingDouble(Line::getY));
+        this.lines = lines;
+        buildBBox();
+    }
+
+
+    public void buildBBox() {
+
+        double minX = Double.POSITIVE_INFINITY;
+        double minY = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+        double maxY = Double.NEGATIVE_INFINITY;
+
+        for (Line line : lines) {
+
+            minX = Math.min(minX, line.getX());
+            minY = Math.min(minY, line.getY());
+            maxX = Math.max(maxX, line.getX() + line.getWidth());
+            maxY = Math.max(maxY, line.getY() + line.getHeight());
+
+        }
+
+        this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
+    }
+
+
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        lines.forEach(line -> sb.append(line.toString()).append("\n"));
+        return sb.toString().trim();
+    }
+
+}
@@ -0,0 +1,50 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+
+@Service
+public class LineBuilderService {
+
+    private static final double CHARACTER_SPACING_DISTANCE_MULTIPLIER = 3.5;
+    private static final double MAX_VERTICAL_CHARACTER_DISTANCE = 0.67;
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+
+
+    public List<Line> buildLines(List<Character> characters, double characterSpacing, double lineSpacing) {
+
+        double maxHorizontalDistance = characterSpacing * CHARACTER_SPACING_DISTANCE_MULTIPLIER;
+        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_CHARACTER_DISTANCE;
+
+        DisjointSets<Character> sets = new DisjointSets<>(characters);
+        AngleFilter filter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
+
+        characters.forEach(character -> {
+            character.getNeighbors().forEach(neighbor -> {
+                double x = neighbor.getHorizontalDistance() / maxHorizontalDistance;
+                double y = neighbor.getVerticalDistance() / maxVerticalDistance;
+                if (filter.matches(neighbor) && Math.pow(x, 2) + Math.pow(y, 2) <= 1) {
+                    sets.union(character, neighbor.getCharacter());
+                }
+            });
+        });
+
+        List<Line> lines = new ArrayList<>();
+        sets.forEach(group -> {
+            List<Character> lineComponents = new ArrayList<>(group);
+            lineComponents.sort(Comparator.comparingDouble(Character::getX));
+            lines.add(new Line(lineComponents, characterSpacing));
+        });
+
+        return lines;
+    }
+
+}
@@ -0,0 +1,78 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
+
+@Service
+public class NearestNeighbourService {
+
+    private static final int NUMBER_OF_NEIGHBOURS = 8;
+    private static final double STEP = 16.0;
+
+
+    public void findNearestNeighbors(List<Character> characters) {
+
+        if (characters.isEmpty()) {
+            return;
+        }
+
+        characters.sort(Comparator.comparingDouble(Character::getX));
+
+        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
+        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
+            maxNeighborCount = characters.size() - 1;
+        }
+
+        for (int i = 0; i < characters.size(); i++) {
+
+            List<Neighbor> candidates = new ArrayList<>();
+
+            int start = i;
+            int end = i + 1;
+
+            double distance = Double.POSITIVE_INFINITY;
+
+            for (double searchDistance = 0; searchDistance < distance; ) {
+
+                searchDistance += STEP;
+                boolean newCandidatesFound = false;
+
+                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
+                    start--;
+                    candidates.add(new Neighbor(characters.get(start), characters.get(i)));
+                    clearLeastDistant(candidates, maxNeighborCount);
+                    newCandidatesFound = true;
+                }
+
+                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
+                    candidates.add(new Neighbor(characters.get(end), characters.get(i)));
+                    clearLeastDistant(candidates, maxNeighborCount);
+                    end++;
+                    newCandidatesFound = true;
+                }
+
+                if (newCandidatesFound && candidates.size() >= maxNeighborCount) {
+                    distance = candidates.get(maxNeighborCount - 1).getDistance();
+                }
+            }
+            clearLeastDistant(candidates, maxNeighborCount);
+            characters.get(i).setNeighbors(new ArrayList<>(candidates));
+        }
+    }
+
+
+    private void clearLeastDistant(List<Neighbor> candidates, int maxNeighborCount) {
+
+        if (candidates.size() > maxNeighborCount) {
+            candidates.sort(Comparator.comparingDouble(Neighbor::getDistance));
+            candidates.remove(candidates.remove(candidates.size() - 1));
+        }
+    }
+
+}
@@ -0,0 +1,99 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
+
+@Service
+public class ReadingOrderService {
+
+    private static final double THRESHOLD = 1;
+
+
+    public List<Zone> resolve(List<Zone> zones, boolean yxOrder) {
+
+        if (zones.isEmpty() || zones.size() == 1) {
+            return zones;
+        }
+
+        if (yxOrder) {
+            zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+            return zones;
+        }
+
+        return resolveMultiColumnReadingOder(zones);
+    }
+
+
+    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
+
+        // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
+        // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
+
+        double minX = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+
+        for (Zone zone : zones) {
+            if (zone.getX() < minX) {
+                minX = zone.getX();
+            }
+            if (zone.getX() + zone.getWidth() > maxX) {
+                maxX = zone.getX() + zone.getWidth();
+            }
+        }
+
+        double midLineXCoordinate = (minX + maxX) / 2;
+
+        List<Zone> leftOf = new ArrayList<>();
+        List<Zone> rightOf = new ArrayList<>();
+        List<Zone> middle = new ArrayList<>();
+        for (Zone zone : zones) {
+            if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
+                leftOf.add(zone);
+            } else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
+                rightOf.add(zone);
+            } else {
+                middle.add(zone);
+            }
+        }
+
+        leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+
+        rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+
+        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+
+        List<Zone> sortedZones = new ArrayList<>();
+        sortedZones.addAll(leftOf);
+        sortedZones.addAll(rightOf);
+
+        ListIterator<Zone> itty = middle.listIterator();
+
+        while (itty.hasNext()) {
+            Zone current = itty.next();
+            for (int i = 0; i < sortedZones.size(); i++) {
+                if (current.getY() < sortedZones.get(i).getY()) {
+                    sortedZones.add(i, current);
+                    itty.remove();
+                    break;
+                }
+            }
+        }
+
+        sortedZones.addAll(middle);
+
+        return sortedZones;
+    }
+
+}
@@ -0,0 +1,56 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Histogram;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
+
+@Service
+public class SpacingService {
+
+    private static final double SPACING_HISTOGRAM_RESOLUTION = 0.5;
+    private static final double SPACING_HISTOGRAM_SMOOTHING_LENGTH = 2.5;
+    private static final double SPACING_HIST_SMOOTHING_STANDARD_DEVIATION = 0.5;
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+
+
+    public double computeCharacterSpacing(List<Character> components) {
+
+        return computeSpacing(components, 0);
+    }
+
+
+    public double computeLineSpacing(List<Character> components) {
+
+        return computeSpacing(components, Math.PI / 2);
+    }
+
+
+    private double computeSpacing(List<Character> components, double angle) {
+
+        double maxDistance = Double.NEGATIVE_INFINITY;
+
+        for (Character component : components) {
+            for (Neighbor neighbor : component.getNeighbors()) {
+                maxDistance = Math.max(maxDistance, neighbor.getDistance());
+            }
+        }
+        Histogram histogram = new Histogram(0, maxDistance, SPACING_HISTOGRAM_RESOLUTION);
+        AngleFilter filter = new AngleFilter(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
+        for (Character component : components) {
+            for (Neighbor neighbor : component.getNeighbors()) {
+                if (filter.matches(neighbor)) {
+                    histogram.add(neighbor.getDistance());
+                }
+            }
+        }
+
+        histogram.gaussianSmooth(SPACING_HISTOGRAM_SMOOTHING_LENGTH, SPACING_HIST_SMOOTHING_STANDARD_DEVIATION);
+        return histogram.getPeakValue();
+    }
+
+}
@@ -0,0 +1,90 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
+
+@Service
+public class ZoneBuilderService {
+
+    private static final double MIN_HORIZONTAL_DISTANCE_MULTIPLIER = -0.5;
+    private static final double MAX_VERTICAL_DISTANCE_MULTIPLIER = 1.2;
+
+    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -3.0;
+
+    private static final double MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER = 0.5;
+
+    private static final double MIN_LINE_SIZE_SCALE = 0.9;
+
+    private static final double MAX_LINE_SIZE_SCALE = 2.5;
+
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+
+    public static final int MAX_ZONES = 300;
+
+
+    public List<Zone> buildZones(List<Line> lines, double characterSpacing, double lineSpacing) {
+
+        double minHorizontalDistance = characterSpacing * MIN_HORIZONTAL_DISTANCE_MULTIPLIER;
+        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_DISTANCE_MULTIPLIER;
+        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
+        double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;
+
+        DisjointSets<Line> sets = new DisjointSets<>(lines);
+
+        double meanHeight = calculateMeanHeight(lines);
+
+        lines.forEach(outerLine -> //
+                lines.forEach(innerLine -> {
+
+                    double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
+                    scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
+
+                    if (!sets.areTogether(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {
+
+                        double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
+                        double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
+
+                        if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance //
+                                || minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
+                            sets.union(outerLine, innerLine);
+                        }
+                    }
+                }));
+
+        List<Zone> zones = new ArrayList<>();
+        sets.forEach(group -> {
+            zones.add(new Zone(new ArrayList<>(group)));
+        });
+
+        if (zones.size() > MAX_ZONES) {
+            List<Line> oneZoneLines = new ArrayList<>();
+            for (Zone zone : zones) {
+                oneZoneLines.addAll(zone.getLines());
+            }
+            return List.of(new Zone(oneZoneLines));
+        }
+
+        return zones;
+    }
+
+
+    private double calculateMeanHeight(List<Line> lines) {
+
+        double meanHeight = 0.0;
+        double weights = 0.0;
+        for (Line line : lines) {
+            double weight = line.getLength();
+            meanHeight += line.getHeight() * weight;
+            weights += weight;
+        }
+        meanHeight /= weights;
+        return meanHeight;
+    }
+
+}
@@ -0,0 +1,18 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils;
+
+public class DoubleUtils {
+
+    public static int compareDouble(double d1, double d2, double precision) {
+
+        if (Double.isNaN(d1) || Double.isNaN(d2)) {
+            return Double.compare(d1, d2);
+        }
+        if (precision == 0) {
+            precision = 1;
+        }
+        long i1 = Math.round(d1 / precision);
+        long i2 = Math.round(d2 / precision);
+        return Long.valueOf(i1).compareTo(i2);
+    }
+
+}
@@ -64,7 +64,7 @@ public class DocumentGraphFactory {

    private void addSections(ClassificationDocument document, Context context) {

-        document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getPageBlocks(), section.getImages(), context));
+        document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getNonEmptyPageBlocks(), section.getImages(), context));
    }


@@ -133,7 +133,7 @@ public class SearchTextWithTextPositionFactory {

    private static void addTextPositionWithFontType(RedTextPosition currentTextPosition, String fontType, List<Integer> fontTypePositions, int stringIdx) {

-        if (currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
+        if (currentTextPosition.getFontName() != null && currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
            fontTypePositions.add(stringIdx);
        }
    }
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -8,6 +9,7 @@ import java.util.Map;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;

@@ -92,28 +92,28 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
     */
    LegacyPDFStreamEngine() throws IOException {

-        addOperator(new BeginText());
-        addOperator(new Concatenate());
-        addOperator(new DrawObject()); // special text version
-        addOperator(new EndText());
-        addOperator(new SetGraphicsStateParameters());
-        addOperator(new Save());
-        addOperator(new Restore());
-        addOperator(new NextLine());
-        addOperator(new SetCharSpacing());
-        addOperator(new MoveText());
-        addOperator(new MoveTextSetLeading());
-        addOperator(new SetFontAndSize());
-        addOperator(new ShowText());
-        addOperator(new ShowTextAdjusted());
-        addOperator(new SetTextLeading());
-        addOperator(new SetMatrix());
-        addOperator(new SetTextRenderingMode());
-        addOperator(new SetTextRise());
-        addOperator(new SetWordSpacing());
-        addOperator(new SetTextHorizontalScaling());
-        addOperator(new ShowTextLine());
-        addOperator(new ShowTextLineAndSpace());
+        addOperator(new BeginText(this));
+        addOperator(new Concatenate(this));
+        addOperator(new DrawObject(this)); // special text version
+        addOperator(new EndText(this));
+        addOperator(new SetGraphicsStateParameters(this));
+        addOperator(new Save(this));
+        addOperator(new Restore(this));
+        addOperator(new NextLine(this));
+        addOperator(new SetCharSpacing(this));
+        addOperator(new MoveText(this));
+        addOperator(new MoveTextSetLeading(this));
+        addOperator(new SetFontAndSize(this));
+        addOperator(new ShowText(this));
+        addOperator(new ShowTextAdjusted(this));
+        addOperator(new SetTextLeading(this));
+        addOperator(new SetMatrix(this));
+        addOperator(new SetTextRenderingMode(this));
+        addOperator(new SetTextRise(this));
+        addOperator(new SetWordSpacing(this));
+        addOperator(new SetTextHorizontalScaling(this));
+        addOperator(new ShowTextLine(this));
+        addOperator(new ShowTextLineAndSpace(this));

        // load additional glyph list for Unicode mapping
        String path = "/org/apache/pdfbox/resources/glyphlist/additional.txt";
@@ -20,6 +20,8 @@ import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
 import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
 import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
 import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
+import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
+import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
 import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
 import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
 import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
@@ -53,10 +55,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private final List<Ruling> graphicsPath = new ArrayList<>();
    @Setter
    protected PDPage pdpage;
-    private int minCharWidth;
-    private int maxCharWidth;
-    private int minCharHeight;
-    private int maxCharHeight;

    private float path_x;
    private float path_y;
@@ -68,26 +66,30 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    public PDFLinesTextStripper() throws IOException {

        super();
-        this.addOperator(new SetStrokingColorSpace());
-        this.addOperator(new SetNonStrokingColorSpace());
-        this.addOperator(new SetLineDashPattern());
-        this.addOperator(new SetStrokingDeviceGrayColor());
-        this.addOperator(new SetNonStrokingDeviceGrayColor());
-        this.addOperator(new SetFlatness());
-        this.addOperator(new SetLineJoinStyle());
-        this.addOperator(new SetLineCapStyle());
-        this.addOperator(new SetStrokingDeviceCMYKColor());
-        this.addOperator(new SetNonStrokingDeviceCMYKColor());
-        this.addOperator(new SetLineMiterLimit());
-        this.addOperator(new SetStrokingDeviceRGBColor());
-        this.addOperator(new SetNonStrokingDeviceRGBColor());
-        this.addOperator(new SetRenderingIntent());
-        this.addOperator(new SetStrokingColor());
-        this.addOperator(new SetNonStrokingColor());
-        this.addOperator(new SetStrokingColorN());
-        this.addOperator(new SetNonStrokingColorN());
-        this.addOperator(new SetFontAndSize());
-        this.addOperator(new SetLineWidth());
+        this.addOperator(new SetStrokingColorSpace(this));
+        this.addOperator(new SetNonStrokingColorSpace(this));
+        this.addOperator(new SetLineDashPattern(this));
+        this.addOperator(new SetStrokingDeviceGrayColor(this));
+        this.addOperator(new SetNonStrokingDeviceGrayColor(this));
+        this.addOperator(new SetFlatness(this));
+        this.addOperator(new SetLineJoinStyle(this));
+        this.addOperator(new SetLineCapStyle(this));
+        this.addOperator(new SetStrokingDeviceCMYKColor(this));
+        this.addOperator(new SetNonStrokingDeviceCMYKColor(this));
+        this.addOperator(new SetLineMiterLimit(this));
+        this.addOperator(new SetStrokingDeviceRGBColor(this));
+        this.addOperator(new SetNonStrokingDeviceRGBColor(this));
+        this.addOperator(new SetRenderingIntent(this));
+        this.addOperator(new SetStrokingColor(this));
+        this.addOperator(new SetNonStrokingColor(this));
+        this.addOperator(new SetStrokingColorN(this));
+        this.addOperator(new SetNonStrokingColorN(this));
+        this.addOperator(new SetFontAndSize(this));
+        this.addOperator(new SetLineWidth(this));
+
+        addOperator(new BeginMarkedContentSequenceWithProperties(this));
+//        addOperator(new BeginMarkedContentSequence(this));
+        addOperator(new EndMarkedContentSequence(this));
    }


@@ -98,6 +100,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

        //move
        switch (operation) {
+
            case OperatorName.MOVE_TO:
                if (arguments.size() == 2) {
                    Point2D.Float pos = transformPosition(floatValue(arguments.get(0)), floatValue(arguments.get(1)));
@@ -219,7 +222,11 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            // This is a quick and dirt hack
            // Happens for file 216.pdf
            log.debug(e.getMessage());
-            return color.getComponents()[0] == 0 && color.getComponents()[1] == 0 && color.getComponents()[2] == 0 && color.getComponents()[1] == 1;
+            var result = true;
+            for (var component : color.getComponents()) {
+                result = result && component == 0;
+            }
+            return result;
        }
    }

@@ -238,33 +245,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                        .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
            }

-            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
-            if (charWidth < minCharWidth) {
-                minCharWidth = charWidth;
-            }
-            if (charWidth > maxCharWidth) {
-                maxCharWidth = charWidth;
-            }
-
-            int charHeight = (int) textPositions.get(i).getHeightDir();
-            if (charHeight < minCharHeight) {
-                minCharHeight = charHeight;
-            }
-            if (charWidth > maxCharHeight) {
-                maxCharHeight = charHeight;
-            }
-
            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
                startIndex++;
                continue;
            }

            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj()) {
+            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -272,9 +261,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {

            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                }
                startIndex = i;
@@ -284,13 +271,10 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                    .getUnicode()
                    .equals("\t")) && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                        .getUnicode()
-                        .equals("\t")))) {
+                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {

                    // Remove false sequence ends (whitespaces)
-                    if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                            .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
+                    if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
                        for (TextPosition t : sublist) {
                            textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                        }
@@ -325,13 +309,29 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    }


+    public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {
+
+        return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
+    }
+
+
+    public boolean checkIfSequenceContainsOnlyWhitespaces(List<TextPosition> sublist) {
+
+        return !(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                .getUnicode()
+                .equals("\t")));
+    }
+
+
+    public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {
+
+        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
+                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
+    }
+
    @Override
    public String getText(PDDocument doc) throws IOException {

-        minCharWidth = Integer.MAX_VALUE;
-        maxCharWidth = 0;
-        minCharHeight = Integer.MAX_VALUE;
-        maxCharHeight = 0;
        textPositionSequences.clear();
        rulings.clear();
        graphicsPath.clear();
@@ -344,3 +344,4 @@ public class PDFLinesTextStripper extends PDFTextStripper {

 }

+
@@ -25,32 +25,26 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.regex.Pattern;

+import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageTree;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.pdmodel.graphics.PDXObject;
 import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
 import org.apache.pdfbox.pdmodel.interactive.pagenavigation.PDThreadBead;
 import org.apache.pdfbox.text.TextPosition;
 import org.apache.pdfbox.text.TextPositionComparator;
-import org.apache.pdfbox.util.QuickSort;
+
+import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;

 /**
 * This is just a copy except i only adjusted lines 594-607 cause this is a bug in Pdfbox.
@@ -63,6 +57,10 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    private static float defaultDropThreshold = 2.5f;
    private static final boolean useCustomQuickSort;

+    @Getter
+    protected final List<PDMarkedContent> markedContents = new ArrayList<>();
+    protected final Deque<PDMarkedContent> currentMarkedContents = new ArrayDeque<>();
+
    private static final Log LOG = LogFactory.getLog(PDFTextStripper.class);

    // enable the ability to set the default indent/drop thresholds
@@ -196,6 +194,45 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    }


+
+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
+    {
+        PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
+        if (this.currentMarkedContents.isEmpty())
+        {
+            this.markedContents.add(markedContent);
+        }
+        else
+        {
+            PDMarkedContent currentMarkedContent =
+                    this.currentMarkedContents.peek();
+            if (currentMarkedContent != null)
+            {
+                currentMarkedContent.addMarkedContent(markedContent);
+            }
+        }
+        this.currentMarkedContents.push(markedContent);
+    }
+
+    @Override
+    public void endMarkedContentSequence()
+    {
+        if (!this.currentMarkedContents.isEmpty())
+        {
+            this.currentMarkedContents.pop();
+        }
+    }
+
+
+    public void xobject(PDXObject xobject)
+    {
+        if (!this.currentMarkedContents.isEmpty())
+        {
+            this.currentMarkedContents.peek().addXObject(xobject);
+        }
+    }
+
+
    /**
     * This will return the text of a document. See writeText. <br>
     * NOTE: The document must not be encrypted when coming into this method.
@@ -877,7 +914,12 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    textList.add(text);
                }
            }
+            if (!this.currentMarkedContents.isEmpty())
+            {
+                this.currentMarkedContents.peek().addText(text);
+            }
        }
+
    }


@@ -1669,7 +1711,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
        int numberOfStrings = line.size();
        for (int i = 0; i < numberOfStrings; i++) {
            WordWithTextPositions word = line.get(i);
-            word.getTextPositions().sort(Comparator.comparing(TextPosition::getX));
+            word.getTextPositions().sort(Comparator.comparing(TextPosition::getXDirAdj));
            writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
            if (i < numberOfStrings - 1) {
                writeWordSeparator();
@@ -2103,6 +2145,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            this.isHangingIndent = true;
        }

+
    }

 }
@@ -0,0 +1,320 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Line2D;
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+import com.knecon.fforesight.service.viewerdoc.ContentStreams;
+import com.knecon.fforesight.service.viewerdoc.model.ColoredLine;
+import com.knecon.fforesight.service.viewerdoc.model.ColoredRectangle;
+import com.knecon.fforesight.service.viewerdoc.model.LayoutGrid;
+import com.knecon.fforesight.service.viewerdoc.model.PlacedText;
+import com.knecon.fforesight.service.viewerdoc.model.Standard14EmbeddableFont;
+import com.knecon.fforesight.service.viewerdoc.model.Visualizations;
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
+public class LayoutGridService {
+
+    ViewerDocumentService viewerDocumentService;
+
+    static float FONT_SIZE = 10f;
+    static float LINE_WIDTH = 1f;
+    static Standard14EmbeddableFont FONT = Standard14EmbeddableFont.helvetica();
+
+    static Color INNER_LINES_COLOR = new Color(255, 175, 175);
+    static Color PARAGRAPH_COLOR = new Color(70, 130, 180);
+    static Color TABLE_COLOR = new Color(102, 205, 170);
+    static Color SECTION_COLOR = new Color(50, 50, 50);
+    static Color HEADLINE_COLOR = new Color(162, 56, 56);
+    static Color HEADER_COLOR = new Color(171, 131, 6);
+    static Color IMAGE_COLOR = new Color(253, 63, 146);
+
+
+    @SneakyThrows
+    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
+    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {
+
+        LayoutGrid layoutGrid = createLayoutGrid(document);
+
+        viewerDocumentService.addVisualizationsOnPage(originFile,
+                destinationFile,
+                Visualizations.builder()
+                        .layer(ContentStreams.KNECON_LAYOUT)
+                        .visualizationsOnPages(layoutGrid.getVisualizationsPerPages())
+                        .layerVisibilityDefaultValue(layerVisibilityDefaultValue)
+                        .build());
+    }
+
+
+    private LayoutGrid createLayoutGrid(Document document) {
+
+        LayoutGrid layoutGrid = new LayoutGrid(document.getNumberOfPages());
+        document.streamAllSubNodes().forEach(semanticNode -> {
+            Color color = switch (semanticNode.getType()) {
+                case PARAGRAPH -> PARAGRAPH_COLOR;
+                case TABLE -> TABLE_COLOR;
+                case SECTION -> SECTION_COLOR;
+                case HEADLINE -> HEADLINE_COLOR;
+                case HEADER, FOOTER -> HEADER_COLOR;
+                case IMAGE -> IMAGE_COLOR;
+                default -> null;
+            };
+            if (isNotSectionOrTableCellOrDocument(semanticNode)) {
+                addAsRectangle(semanticNode, layoutGrid, color);
+            }
+            if (semanticNode.getType().equals(NodeType.SECTION)) {
+                addSection(semanticNode, layoutGrid, color);
+            }
+            if (semanticNode.getType().equals(NodeType.TABLE)) {
+                Table table = (Table) semanticNode;
+                addInnerTableLines(table, layoutGrid);
+            }
+        });
+
+        return layoutGrid;
+    }
+
+
+    private void addInnerTableLines(Table table, LayoutGrid layoutGrid) {
+
+        if (table.getNumberOfCols() < 1 || table.getNumberOfRows() < 1) {
+            return;
+        }
+        for (Page page : table.getPages()) {
+            Optional<Integer> optionalFirstRowOnPage = table.streamCol(0).filter(tableCell -> tableCell.isOnPage(page.getNumber())).map(TableCell::getRow).findFirst();
+            if (optionalFirstRowOnPage.isEmpty()) {
+                continue;
+            }
+            int firstRowOnPage = optionalFirstRowOnPage.get();
+            Stream<Double> xStream = switch (page.getRotation()) {
+                case 90 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinX);
+                case 180 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMaxX);
+                case 270 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMaxX);
+                default -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMinX);
+            };
+            List<Double> xs = xStream.collect(Collectors.toList());
+            xs.remove(0);
+            Stream<Double> yStream = switch (page.getRotation()) {
+                case 90 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMinY);
+                case 180 -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMinY);
+                case 270 -> streamBBoxOfCellsOnPage(table.streamRow(firstRowOnPage), page).map(RectangularShape::getMaxY);
+                default -> streamBBoxOfCellsOnPage(table.streamCol(0), page).map(RectangularShape::getMaxY);
+            };
+            List<Double> ys = yStream.collect(Collectors.toList());
+            ys.remove(0);
+
+            Rectangle2D tableBBox = table.getBBox().get(table.getFirstPage());
+            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredLines();
+            xs.forEach(x -> {
+                Line2D line = new Line2D.Double(new Point2D.Double(x, tableBBox.getMaxY()), new Point2D.Double(x, tableBBox.getMinY()));
+                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR, LINE_WIDTH));
+            });
+            ys.forEach(y -> {
+                Line2D line = new Line2D.Double(new Point2D.Double(tableBBox.getMinX(), y), new Point2D.Double(tableBBox.getMaxX(), y));
+                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR, LINE_WIDTH));
+            });
+        }
+    }
+
+
+    private static Stream<Rectangle2D> streamBBoxOfCellsOnPage(Stream<TableCell> table, Page page) {
+
+        return table.filter(tableCell -> tableCell.isOnPage(page.getNumber())).map(TableCell::getBBox).map(bBoxMap -> bBoxMap.get(page));
+    }
+
+
+    private void addSection(SemanticNode semanticNode, LayoutGrid layoutGrid, Color color) {
+
+        Map<Page, Rectangle2D> bBoxMap = semanticNode.getBBox();
+        List<SemanticNode> subSections = semanticNode.streamAllSubNodesOfType(NodeType.SECTION).toList();
+        Page firstPage = semanticNode.getFirstPage();
+        if (!subSections.isEmpty()) {
+            addPlacedText(firstPage, bBoxMap.get(firstPage), buildTreeIdString(semanticNode), layoutGrid);
+        } else {
+            bBoxMap.forEach(((page, textBBox) -> addPlacedText(page, textBBox, buildTreeIdString(semanticNode), layoutGrid)));
+        }
+        if (bBoxMap.values().size() == 1) {
+            Rectangle2D r = RectangleTransformations.pad(bBoxMap.get(firstPage), LINE_WIDTH, LINE_WIDTH);
+            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(firstPage.getNumber() - 1).getColoredLines();
+            List<Line2D> lines = createLinesFromRectangle(r, firstPage.getRotation());
+            // add string to top line
+            var firstLine = lines.remove(0);
+            coloredLines.add(new ColoredLine(firstLine, color, LINE_WIDTH));
+            for (Line2D line : lines) {
+                coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+            }
+            return;
+        }
+        List<Page> pagesInOrder = bBoxMap.keySet().stream().sorted(Comparator.comparingInt(Page::getNumber)).collect(Collectors.toList());
+        pagesInOrder.remove(0);
+        addLinesForFirstPageOfSection(semanticNode, color, firstPage, layoutGrid);
+        var lastPage = pagesInOrder.remove(pagesInOrder.size() - 1);
+        addLinesForLastPageOfSection(semanticNode, color, lastPage, layoutGrid);
+        for (Page middlePage : pagesInOrder) {
+            addLinesForMiddlePageOfSection(semanticNode, color, middlePage, layoutGrid);
+        }
+    }
+
+
+    @SneakyThrows
+    private void addPlacedText(Page page, Rectangle2D textBBox, String s, LayoutGrid layoutGrid) {
+
+        Point2D.Float upperLeftCorner = switch (page.getRotation()) {
+            case 90 -> new Point2D.Float((float) (textBBox.getMinX()), (float) textBBox.getMinY());
+            case 180 -> new Point2D.Float((float) (textBBox.getMaxX()), (float) textBBox.getMinY());
+            case 270 -> new Point2D.Float((float) (textBBox.getMaxX()), (float) textBBox.getMaxY());
+            default -> new Point2D.Float((float) (textBBox.getMinX()), (float) textBBox.getMaxY());
+        };
+        var placedTexts = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getPlacedTexts();
+        upperLeftCorner.setLocation(upperLeftCorner.getX() - ((FONT.getStringWidth(s) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), upperLeftCorner.getY() - FONT_SIZE);
+        placedTexts.add(PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, Color.BLACK, FONT));
+    }
+
+
+    private void addLinesForMiddlePageOfSection(SemanticNode semanticNode, Color color, Page middlePage, LayoutGrid layoutGrid) {
+
+        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(middlePage.getNumber() - 1).getColoredLines();
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(middlePage), LINE_WIDTH, LINE_WIDTH);
+        var midPageLines = createLinesFromRectangle(r, middlePage.getRotation());
+        // remove top line
+        midPageLines.remove(0);
+        // remove top line
+        midPageLines.remove(1);
+        // add string to left line
+        var leftLine = midPageLines.remove(1);
+        coloredLines.add(new ColoredLine(leftLine, color, LINE_WIDTH));
+        for (Line2D line : midPageLines) {
+            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private void addLinesForLastPageOfSection(SemanticNode semanticNode, Color color, Page lastPage, LayoutGrid layoutGrid) {
+
+        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(lastPage.getNumber() - 1).getColoredLines();
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(lastPage), LINE_WIDTH, LINE_WIDTH);
+        var lastPageLines = createLinesFromRectangle(r, lastPage.getRotation());
+        // remove top line
+        lastPageLines.remove(0);
+        // add string to left line
+        var leftLine = lastPageLines.remove(2);
+        coloredLines.add(new ColoredLine(leftLine, color, LINE_WIDTH));
+        for (Line2D line : lastPageLines) {
+            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private void addLinesForFirstPageOfSection(SemanticNode semanticNode, Color color, Page firstPage, LayoutGrid layoutGrid) {
+
+        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(firstPage.getNumber() - 1).getColoredLines();
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(firstPage), LINE_WIDTH, LINE_WIDTH);
+        var firstPageLines = createLinesFromRectangle(r, firstPage.getRotation());
+        // remove bottom line
+        firstPageLines.remove(2);
+        // add string to top line
+        var firstLine = firstPageLines.remove(0);
+        coloredLines.add(new ColoredLine(firstLine, color, LINE_WIDTH));
+        for (Line2D line : firstPageLines) {
+            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private String buildTreeIdString(SemanticNode semanticNode) {
+
+        return semanticNode.getTreeId().stream().map(Object::toString).collect(Collectors.joining("."));
+    }
+
+
+    /*
+    A __________________ B
+     |                  |
+     |                  |
+     |                  |
+     |                  |
+    D|__________________| C
+    The returned List are the lines [AB, BC, DC, AD]
+    The List is reordered, such that the order of the returned lines are always as viewed on the page.
+     */
+    private List<Line2D> createLinesFromRectangle(Rectangle2D r, int pageRotation) {
+        // +0.5 to join the lines
+        List<Line2D> lines = new ArrayList<>(4);
+        float lineWidthCorrection = LINE_WIDTH * 0.5f;
+        Point2D.Float a = new Point2D.Float((float) r.getMinX(), (float) r.getMaxY());
+        Point2D.Float a1 = new Point2D.Float((float) r.getMinX() - lineWidthCorrection, (float) r.getMaxY());
+        Point2D.Float b = new Point2D.Float((float) r.getMaxX(), (float) r.getMaxY());
+        Point2D.Float b1 = new Point2D.Float((float) r.getMaxX() + lineWidthCorrection, (float) r.getMaxY());
+        Point2D.Float c = new Point2D.Float((float) r.getMaxX(), (float) r.getMinY());
+        Point2D.Float c1 = new Point2D.Float((float) r.getMaxX() + lineWidthCorrection, (float) r.getMinY());
+        Point2D.Float d = new Point2D.Float((float) r.getMinX(), (float) r.getMinY());
+        Point2D.Float d1 = new Point2D.Float((float) r.getMinX() - lineWidthCorrection, (float) r.getMinY());
+        lines.add(new Line2D.Float(a1, b1));
+        lines.add(new Line2D.Float(b, c));
+        lines.add(new Line2D.Float(d1, c1));
+        lines.add(new Line2D.Float(a, d));
+
+        return switch (pageRotation) {
+            case 90 -> {
+                Collections.rotate(lines, 1);
+                yield lines;
+            }
+            case 180 -> {
+                Collections.rotate(lines, 2);
+                yield lines;
+            }
+            case 270 -> {
+                Collections.rotate(lines, 3);
+                yield lines;
+            }
+
+            default -> lines;
+        };
+    }
+
+
+    private static boolean isNotSectionOrTableCellOrDocument(SemanticNode semanticNode) {
+
+        return !(semanticNode.getType().equals(NodeType.DOCUMENT) || semanticNode.getType().equals(NodeType.SECTION) || semanticNode.getType().equals(NodeType.TABLE_CELL));
+    }
+
+
+    private void addAsRectangle(SemanticNode semanticNode, LayoutGrid layoutGrid, Color color) {
+
+        semanticNode.getBBox()
+                .forEach((page, textBBox) -> layoutGrid.getVisualizationsPerPages()
+                        .get(page.getNumber() - 1)
+                        .getColoredRectangles()
+                        .add(new ColoredRectangle(textBBox, color, LINE_WIDTH)));
+    }
+
+}
@@ -0,0 +1,270 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
+import org.apache.pdfbox.util.Matrix;
+import org.springframework.core.io.ClassPathResource;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class PdfDraw {
+
+    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) throws IOException {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
+                        pageNumber,
+                        rectanglesPerPage.get(pageNumber - 1),
+                        PdfVisualisationUtility.Options.builder().stroke(true).build());
+            }
+            pdDocument.save(out);
+        }
+
+    }
+
+
+    public static void drawRectanglesPerPageNumberedByLine(String filename, List<List<List<Rectangle2D>>> rectanglesPerPage, String tmpFileName) throws IOException {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                var rectanglesOnPage = rectanglesPerPage.get(pageNumber - 1);
+                for (int lineNumber = 0; lineNumber < rectanglesOnPage.size(); lineNumber++) {
+                    var rectanglesInLine = rectanglesOnPage.get(lineNumber);
+                    PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, rectanglesInLine, PdfVisualisationUtility.Options.builder().stroke(true).build());
+                    double y = Math.min(rectanglesInLine.get(0).getMinY(), rectanglesInLine.get(0).getMaxY());
+                    PdfVisualisationUtility.drawText(String.format("%d", lineNumber),
+                            pdDocument,
+                            new Point2D.Double(rectanglesInLine.get(0).getX() - (5 + (5 * countNumberOfDigits(lineNumber))), y + 2),
+                            pageNumber,
+                            PdfVisualisationUtility.Options.builder().stroke(true).build());
+                }
+            }
+            pdDocument.save(out);
+        }
+
+    }
+
+
+    private static int countNumberOfDigits(int num) {
+
+        int final_num = num;
+        if (final_num == 0) {
+            return 1;
+        }
+        int count = 0;
+        for (; final_num != 0; final_num /= 10) {
+            count++;
+        }
+        return count;
+    }
+
+
+    public static void drawDocumentGraph(PDDocument document, Document documentGraph) {
+
+        documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry));
+    }
+
+
+    public static void drawNode(PDDocument document, DocumentTree.Entry entry) {
+
+        Options options = buildStandardOptionsForNodes(entry);
+
+        drawBBoxAndLabelAndNumberOnPage(document, entry, options);
+
+    }
+
+
+    public static void drawTextBlock(PDDocument document, TextBlock textBlock, Options options) {
+
+        textBlock.getAtomicTextBlocks().forEach(atb -> drawAtomicTextBlock(document, atb, options));
+    }
+
+
+    public static void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, Options options) {
+
+        drawRectangle2DList(document, atomicTextBlock.getPage().getNumber(), atomicTextBlock.getPositions().stream().toList(), options);
+
+    }
+
+
+    @SneakyThrows
+    private static void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, Options options, boolean rotate) {
+
+        var pdPage = document.getPage(pageNumber - 1);
+        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
+
+        contentStream.setNonStrokingColor(options.getStrokeColor());
+        contentStream.setLineWidth(options.getStrokeWidth());
+
+        contentStream.beginText();
+        if (rotate) {
+            contentStream.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(15), (float) location.getX(), (float) location.getY()));
+        } else {
+            contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
+        }
+        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
+        contentStream.showText(string);
+        contentStream.endText();
+        contentStream.close();
+    }
+
+
+    @SneakyThrows
+    public static void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, Options options) {
+
+        var pdPage = document.getPage(pageNumber - 1);
+        drawRectangle2DList(document, rectCollection, options, pdPage);
+    }
+
+
+    private static void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, Options options, PDPage pdPage) throws IOException {
+
+        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
+
+        contentStream.setStrokingColor(options.getStrokeColor());
+        contentStream.setNonStrokingColor(options.getFillColor());
+        contentStream.setLineWidth(options.getStrokeWidth());
+
+        for (var r : rectCollection) {
+            contentStream.addRect((float) r.getMinX(), (float) r.getMinY(), (float) r.getWidth(), (float) r.getHeight());
+
+            if (options.isStroke() && options.isFill()) {
+                contentStream.fillAndStroke();
+            } else if (options.isStroke()) {
+                contentStream.stroke();
+            } else if (options.isFill()) {
+                contentStream.fill();
+            }
+        }
+        contentStream.close();
+    }
+
+
+    @SneakyThrows
+    public static void drawRectanglesAndLinesPerPage(String filename, List<List<Rectangle2D>> list, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+//                PdfVisualisationUtility.drawLine2DList(pdDocument,
+//                        pageNumber,
+//                        list.get(pageNumber - 1),
+//                        PdfVisualisationUtility.Options.builder().stroke(true).build());
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
+                        pageNumber,
+                        rectanglesPerPage.get(pageNumber - 1),
+                        PdfVisualisationUtility.Options.builder().stroke(true).build());
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().stroke(true).build());
+            }
+            pdDocument.save(out);
+        }
+    }
+
+
+    @SneakyThrows
+    public static void drawLinesPerPage(String filename, List<List<Ruling>> linesPerPage, String tmpFileName) {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                PdfVisualisationUtility.drawLine2DList(pdDocument,
+                        pageNumber,
+                        linesPerPage.get(pageNumber - 1),
+                        PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
+            }
+            pdDocument.save(out);
+        }
+    }
+
+
+    @Builder
+    @AllArgsConstructor
+    @Getter
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    public static class Options {
+
+        boolean stroke;
+        @Builder.Default
+        Color strokeColor = Color.BLACK;
+        @Builder.Default
+        float strokeWidth = 1f;
+
+        boolean fill;
+        @Builder.Default
+        Color fillColor = Color.BLACK;
+
+    }
+
+
+    private static Options buildStandardOptionsForNodes(DocumentTree.Entry entry) {
+
+        return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
+            case DOCUMENT -> Color.LIGHT_GRAY;
+            case HEADER, FOOTER -> Color.GREEN;
+            case PARAGRAPH -> Color.BLUE;
+            case HEADLINE -> Color.RED;
+            case SECTION -> Color.BLACK;
+            case TABLE -> Color.ORANGE;
+            case TABLE_CELL -> Color.GRAY;
+            case IMAGE -> Color.MAGENTA;
+        }).build();
+    }
+
+
+    private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, Options options) {
+
+        Map<Page, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
+        for (Page page : rectanglesPerPage.keySet()) {
+            Rectangle2D rectangle2D = rectanglesPerPage.get(page);
+            if (entry.getType() == NodeType.SECTION) {
+                rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
+            }
+            drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options);
+            drawText(buildString(entry),
+                    document,
+                    new Point2D.Double(rectangle2D.getMinX(), rectangle2D.getMaxY() + 2),
+                    page.getNumber(),
+                    options,
+                    entry.getType() == NodeType.TABLE_CELL);
+        }
+    }
+
+
+    private static String buildString(DocumentTree.Entry entry) {
+
+        return entry.getNode().getNumberOnPage() + ": " + entry.getTreeId() + ": " + entry.getType();
+    }
+
+}
@@ -0,0 +1,56 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import lombok.experimental.UtilityClass;
+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.text.TextPosition;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@UtilityClass
+public class MarkedContentUtils {
+
+    public static final String HEADER = "Header";
+    public static final String FOOTER = "Footer";
+
+    public List<Rectangle2D> getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {
+
+        if (markedContents == null) {
+            return Collections.emptyList();
+        }
+
+        var markedContentByYPosition = markedContents.stream()
+                .filter(m -> m.getTag().equals("Artifact"))
+                .filter(m -> m.getProperties() != null)
+                .filter(m -> m.getProperties().getItem("Subtype") != null)
+                .filter(m -> ((COSName) m.getProperties().getItem("Subtype")).getName().equals(subtype))
+                .map(PDMarkedContent::getContents).flatMap(Collection::stream)
+                .filter(t -> t instanceof TextPosition)
+                .map(t -> (TextPosition) t)
+                .filter(t -> !t.getUnicode().equals(" "))
+                .collect(Collectors.groupingBy(TextPosition::getY));
+
+        if (markedContentByYPosition.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        return markedContentByYPosition.values().stream()
+                .map(textPositions -> new TextPositionSequence(textPositions.stream()
+                        .toList(), 0, true)
+                        .getRectangle())
+                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList());
+    }
+
+
+    public boolean intersects(TextPageBlock textBlock, Map<String, List<Rectangle2D>> markedContentBboxPerType, String type) {
+        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).stream().anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
+    }
+
+}
@@ -12,6 +12,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.apache.pdfbox.pdmodel.font.Standard14Fonts;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
@@ -71,7 +72,7 @@ public class PdfVisualisationUtility {

        contentStream.beginText();
        contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
-        contentStream.setFont(PDType1Font.HELVETICA, 10);
+        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
        contentStream.showText(string);
        contentStream.endText();
        contentStream.close();
@@ -19,9 +19,10 @@ public final class PositionUtils {

        double threshold = textBlock.getMostPopularWordHeight() * 3;

-        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX() && textBlock.getPdfMaxX() - threshold < btf.getTopLeft()
-                .getX() + btf.getWidth() && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY() && textBlock.getPdfMaxY() - threshold < btf.getTopLeft()
-                .getY() + btf.getHeight()) {
+        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX()
+                && textBlock.getPdfMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth()
+                && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY()
+                && textBlock.getPdfMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
            return true;
        } else {
            return false;
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 import java.util.List;
 import java.util.stream.Collectors;

-import org.apache.pdfbox.util.QuickSort;

 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
@@ -3,40 +3,53 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
    application
    id("com.knecon.fforesight.java-conventions")
-    id("org.springframework.boot") version "3.1.2"
-    id("io.spring.dependency-management") version "1.1.2"
+    id("org.springframework.boot") version "3.1.5"
+    id("io.spring.dependency-management") version "1.1.3"
    id("org.sonarqube") version "4.3.0.3225"
-    id("io.freefair.lombok") version "8.1.0"
+    id("io.freefair.lombok") version "8.4"
 //    id("org.graalvm.buildtools.native") version "0.9.23"
 }

 description = "layoutparser-service-server"

+configurations {
+    all {
+        exclude(group = "commons-logging", module = "commons-logging")
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
+        exclude(group = "com.iqser.red.commons", module = "logging-commons")
+    }
+}
+
+
 val jacksonVersion = "2.15.2"
-val pdfBoxVersion = "3.0.0-alpha2"
+val pdfBoxVersion = "3.0.0"
+val springBootStarterVersion = "3.1.5"

 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.commons:storage-commons:2.29.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
-    implementation("com.amazonaws:aws-java-sdk-s3:1.12.526")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
+    implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
+    implementation("net.logstash.logback:logstash-logback-encoder:7.4")

    // for integration testing only
+    testImplementation(project(":viewer-doc-processor"))
    testImplementation(project(":layoutparser-service-internal-api"))

-    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
-    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.124.0")
+    testImplementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
+    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0")
    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
-    testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
 }

 /*
@@ -71,10 +84,8 @@ sonarqube {

 tasks.named<BootBuildImage>("bootBuildImage") {

-    println("--------------------" + layout.projectDirectory.file( "src/main/resources/Aptfile").toString());
-    var aptfilePath = layout.projectDirectory.file( "src/main/resources/Aptfile").toString() // make sure this path is valid because there will be no log or exception if not
-    bindings.add("${aptfilePath}:/workspace/Aptfile:ro")
-    buildpacks.set(listOf("ghcr.io/fagiani/buildpacks/fagiani_apt@sha256:6471c8c70f32b749e29f65ae562ac0339fecad26aa9217628c00a6c31f197dae", "urn:cnb:builder:paketo-buildpacks/java"))
+    environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
+    environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")

    imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
    if (project.hasProperty("buildbootDockerHostNetwork")) {
@@ -5,6 +5,7 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
@@ -13,8 +14,11 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingService
 import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.aop.ObservedAspect;
+
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
-@Import({MetricsConfiguration.class, StorageAutoConfiguration.class,  LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
+@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

@@ -23,4 +27,11 @@ public class Application {
        SpringApplication.run(Application.class, args);
    }

+
+    @Bean
+    public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
+
+        return new ObservedAspect(observationRegistry);
+    }
+
 }
@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.queue;

+import java.util.Arrays;
+
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
@@ -45,12 +47,12 @@ public class MessageHandler {
        }
        LayoutParsingFinishedEvent layoutParsingFinishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent);
-        log.info("Layout parsing finished {} in {} ms", layoutParsingRequest.identifier(), layoutParsingFinishedEvent.duration());
    }


    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {

+        Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
    }

@@ -24,8 +24,7 @@ public class MessagingConfiguration {
    @Bean
    public Queue layoutparsingResponseQueue() {

-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
    }


@@ -1,11 +0,0 @@
-# you can list packages
-ghostscript
-cabextract
-xfonts-utils
-fonts-liberation
-
-# or include links to specific .deb files
-http://ftp.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb
-
-# or add custom apt repos (only required if using packages outside of the standard Ubuntu APT repositories)
-# :repo:deb http://cz.archive.ubuntu.com/ubuntu artful main universe
@@ -5,10 +5,19 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/

 fforesight.tenants.remote: true

+logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
+
+logging.type: ${LOGGING_TYPE:CONSOLE}
+kubernetes.namespace: ${NAMESPACE:default}
+project.version: 1.0-SNAPSHOT
+
+
 server:
  port: 8080

 spring:
+  application:
+    name: layoutparser-service
  main:
    allow-circular-references: true # FIXME
  rabbitmq:
@@ -32,6 +41,13 @@ management:
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
  endpoints.web.exposure.include: prometheus, health
+  tracing:
+    enabled: ${TRACING_ENABLED:false}
+    sampling:
+      probability: ${TRACING_PROBABILITY:1.0}
+  otlp:
+    tracing:
+      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}


 storage:
@@ -0,0 +1,17 @@
+<configuration>
+
+    <springProperty scope="configuration" name="logType" source="logging.type"/>
+    <springProperty scope="context" name="application.name" source="spring.application.name"/>
+    <springProperty scope="context" name="version" source="project.version"/>
+    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+
+    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="${logType}"/>
+    </root>
+
+</configuration>
@@ -1,11 +1,11 @@
 package com.knecon.fforesight.service.layoutparser.server;

+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
 import java.awt.Color;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -17,6 +17,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
@@ -43,13 +44,25 @@ public class BdrJsonBuildTest extends AbstractTest {


    @SneakyThrows
-    protected Document buildGraph(File filename) {
+    protected Document buildGraph(File file) {
+
+        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                file,
+                new ImageServiceResponse(),
+                new TableServiceResponse(),
+                file.toString()));
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testBDRFile199865() {
+
+        File file = new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile();
+        Document document = buildGraph(file);
+        ResearchDocumentData researchDocumentData = TaasDocumentDataMapper.fromDocument(document);
+        assertThat(researchDocumentData).isNotNull();

-        try (InputStream inputStream = new FileInputStream(filename)) {
-            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
-                return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS, pdDocument, new ImageServiceResponse(), new TableServiceResponse()));
-            }
-        }
    }


@@ -96,10 +109,7 @@ public class BdrJsonBuildTest extends AbstractTest {

    private static void visualizeSemanticNodes(File file, File resultingFileName, Document document, TextBlock textBlock) throws IOException {

-        try (var fileStream = new FileInputStream(file);//
-             PDDocument pdDocument = Loader.loadPDF(fileStream);//
-             var outputStream = new FileOutputStream(resultingFileName)//
-        ) {
+        try (PDDocument pdDocument = Loader.loadPDF(file); var outputStream = new FileOutputStream(resultingFileName)) {
            PdfDraw.drawDocumentGraph(pdDocument, document);
            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
            pdDocument.save(outputStream);
@@ -6,7 +6,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

-import org.apache.pdfbox.Loader;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -30,10 +29,10 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;

@@ -96,9 +95,10 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(pdfFileResource.getInputStream()),
+                pdfFileResource.getFile(),
                new ImageServiceResponse(),
-                new TableServiceResponse()));
+                new TableServiceResponse(),
+                filePath));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -148,6 +148,7 @@ public class HeadlinesGoldStandardIntegrationTest {
    @EqualsAndHashCode
    @AllArgsConstructor
    @ToString
+    @SuppressWarnings("PMD.AvoidFieldNameMatchingTypeName")
    private class Headline {

        private int page;
@@ -0,0 +1,36 @@
+package com.knecon.fforesight.service.layoutparser.server;
+
+import java.util.Arrays;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class LayoutparserEnd2EndTest extends AbstractTest {
+
+    @Autowired
+    private LayoutParsingPipeline layoutParsingPipeline;
+
+
+    @Test
+    @SneakyThrows
+    public void testLayoutParserEndToEnd() {
+
+        String s = "（";
+        String s1 = "）";
+        prepareStorage("files/Minimal Examples/WrongOrderPage1.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
+        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
+    }
+
+}
@@ -1,58 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server.graph;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.InputStream;
-
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
-
-import lombok.SneakyThrows;
-
-public class BuildDocumentGraphTest extends AbstractTest {
-
-    @Autowired
-    protected LayoutParsingPipeline layoutParsingPipeline;
-
-
-    @Test
-    @Disabled
-    public void buildMetolachlor() {
-
-        Document documentGraph = buildGraph("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
-        assertEquals(221, documentGraph.getPages().size());
-        assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
-        assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
-    }
-
-
-    @SneakyThrows
-    protected Document buildGraph(String filename) {
-
-        if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
-            prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json");
-        } else {
-            prepareStorage(filename);
-        }
-        ClassPathResource fileResource = new ClassPathResource(filename);
-
-        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream)) {
-            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
-                    pdDocument,
-                    layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
-                    new TableServiceResponse()));
-        }
-    }
-
-}
@@ -6,17 +6,21 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;

+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;

 import lombok.SneakyThrows;

-public class DocumentDataTests extends BuildDocumentGraphTest{
+public class DocumentDataTests extends BuildDocumentTest {
+
    @Test
+    @Disabled // This test takes waaaaaay too long, it's ridiculous
    @SneakyThrows
    public void createDocumentDataForAllFiles() {

@@ -35,11 +39,12 @@ public class DocumentDataTests extends BuildDocumentGraphTest{
        for (String pdfFileName : pdfFileNames) {
            System.out.println(pdfFileName);
            DocumentData documentData = DocumentDataMapper.toDocumentData(buildGraph(resource.getFile().toPath().getParent().relativize(Path.of(pdfFileName)).toString()));
-                        File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
+            File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
            outputFile.toPath().getParent().toFile().mkdirs();
            try (var out = new FileOutputStream(outputFile.toString().replace(".pdf", ".json"))) {
                ObjectMapperFactory.create().writeValue(out, documentData);
            }
        }
    }
+
 }
@@ -1,12 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;

-import org.apache.pdfbox.Loader;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;
@@ -15,15 +13,16 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;

 import lombok.SneakyThrows;

-public class DocumentGraphJsonWritingTest extends BuildDocumentGraphTest {
+public class DocumentGraphJsonWritingTest extends BuildDocumentTest {

    @Test
    @SneakyThrows
@@ -56,9 +55,10 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentGraphTest {
    private void writeJsons(Path filename) {

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(new FileInputStream(filename.toFile())),
+                filename.toFile(),
                new ImageServiceResponse(),
-                new TableServiceResponse()));
+                new TableServiceResponse(),
+                filename.toFile().toString()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -16,11 +16,12 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Ta
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentGraphMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.tenantcommons.TenantContext;

 import lombok.SneakyThrows;

-public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
+public class DocumentGraphMappingTest extends BuildDocumentTest {

    @Test
    @SneakyThrows
@@ -13,18 +13,21 @@ import org.springframework.core.io.ClassPathResource;

 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;

-public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
+@Slf4j
+public class DocumentGraphVisualizationTest extends BuildDocumentTest {

    @Test
    @SneakyThrows
-    @Disabled
+//    @Disabled
    public void visualizeMetolachlor() {

-        String filename = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
+        String filename = "files/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (4).pdf";
        visualizePdf(filename);
    }

@@ -44,7 +47,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
    @Disabled
    public void visualizeCraftedDocument() {

-        String filename = "files/crafted document.pdf";
+        String filename = "files/1 Abamectin_prr.pdf";
        visualizePdf(filename);
    }

@@ -64,12 +67,13 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
        File tmpFile = new File("/tmp/" + Path.of(filename).getFileName().toString() + "_SEMANTIC_NODES_BBOX.pdf");
        ClassPathResource fileResource = new ClassPathResource(filename);

-        try (var fileStream = fileResource.getInputStream();//
-             PDDocument pdDocument = Loader.loadPDF(fileStream)//
-        ) {
-                PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
-                PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
-                pdDocument.save(tmpFile);
+        try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) {
+            log.info("drawing document");
+            PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
+            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
+            log.info("saving document");
+            pdDocument.save(tmpFile);
+            log.info("saved document");
        }
    }

@@ -0,0 +1,203 @@
+package com.knecon.fforesight.service.layoutparser.server.graph;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import lombok.SneakyThrows;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.cos.COSArray;
+import org.apache.pdfbox.cos.COSBase;
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSNumber;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureElement;
+import org.apache.pdfbox.pdmodel.documentinterchange.logicalstructure.PDStructureNode;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.text.PDFMarkedContentExtractor;
+import org.apache.pdfbox.text.TextPosition;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;
+
+/**
+ * @author mkl
+ */
+@Disabled
+public class ExtractMarkedContentTest {
+    final static File RESULT_FOLDER = new File("target/test-outputs", "extract");
+
+    @BeforeEach
+    public void setUpBeforeClass() throws Exception {
+        RESULT_FOLDER.mkdirs();
+    }
+
+    /**
+     * <a href="https://stackoverflow.com/questions/54956720/how-to-replace-a-space-with-a-word-while-extract-the-data-from-pdf-using-pdfbox">
+     * How to replace a space with a word while extract the data from PDF using PDFBox
+     * </a>
+     * <br/>
+     * <a href="https://drive.google.com/open?id=10ZkdPlGWzMJeahwnQPzE6V7s09d1nvwq">
+     * test.pdf
+     * </a> as "testWPhromma.pdf"
+     * <p>
+     * This test shows how to, in principle, extract tagged text.
+     * </p>
+     */
+    @Test
+    @SneakyThrows
+    public void testExtractTestWPhromma() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "testWPhromma.pdf");
+        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+            document.close();
+        }
+    }
+
+    /**
+     * <a href="https://stackoverflow.com/questions/59192443/get-tags-related-bboxs-even-though-there-is-no-attributes-a-in-document-cata">
+     * Get tag's related BBox's even though there is no attributes (/A in document catalog structure) related to Layout in PDFBox?
+     * </a>
+     * <br/>
+     * <a href="https://drive.google.com/file/d/1_-tuWuReaTvrDsqQwldTnPYrMHSpXIWp/view?usp=sharing">
+     * res_multipage.pdf
+     * </a>
+     * <p>
+     * This test shows how to, in principle, extract tagged text from this document.
+     * </p>
+     */
+    @Test
+    public void testExtractResMultipage() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "res_multipage.pdf");
+
+        try(PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+            document.close();
+        }
+    }
+
+    /**
+     * <a href="https://issues.apache.org/jira/browse/PDFBOX-5613">
+     * PDFBOX-5613 - uncorrent paragraph split
+     * </a>
+     * <br/>
+     * <a href="https://issues.apache.org/jira/secure/attachment/13058595/Daily%20Report.pdf">
+     * Daily Report.pdf
+     * </a>
+     * <p>
+     * Making use of the marked content, the contents of the file can better be extracted.
+     * </p>
+     */
+    @Test
+    public void testExtractDailyReport() throws IOException {
+        System.out.printf("\n\n===\n%s\n===\n", "Daily Report.pdf");
+        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+
+            Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
+
+            for (PDPage page : document.getPages()) {
+                PDFMarkedContentExtractor extractor = new PDFMarkedContentExtractor();
+                extractor.processPage(page);
+
+                Map<Integer, PDMarkedContent> theseMarkedContents = new HashMap<>();
+                markedContents.put(page, theseMarkedContents);
+                for (PDMarkedContent markedContent : extractor.getMarkedContents()) {
+                    theseMarkedContents.put(markedContent.getMCID(), markedContent);
+                }
+            }
+
+            PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
+            showStructure(root, markedContents);
+            document.close();
+        }
+    }
+
+    /**
+     * @see #testExtractTestWPhromma()
+     */
+    void showStructure(PDStructureNode node, Map<PDPage, Map<Integer, PDMarkedContent>> markedContents) {
+        String structType = null;
+        PDPage page = null;
+        if (node instanceof PDStructureElement) {
+            PDStructureElement element = (PDStructureElement) node;
+            structType = element.getStructureType();
+            page = element.getPage();
+        }
+        Map<Integer, PDMarkedContent> theseMarkedContents = markedContents.get(page);
+        System.out.printf("<%s>\n", structType);
+        for (Object object : node.getKids()) {
+            if (object instanceof COSArray) {
+                for (COSBase base : (COSArray) object) {
+                    if (base instanceof COSDictionary) {
+                        showStructure(PDStructureNode.create((COSDictionary) base), markedContents);
+                    } else if (base instanceof COSNumber) {
+                        showContent(((COSNumber)base).intValue(), theseMarkedContents);
+                    } else {
+                        System.out.printf("?%s\n", base);
+                    }
+                }
+            } else if (object instanceof PDStructureNode) {
+                showStructure((PDStructureNode) object, markedContents);
+            } else if (object instanceof Integer) {
+                showContent((Integer)object, theseMarkedContents);
+            } else {
+                System.out.printf("?%s\n", object);
+            }
+
+        }
+        System.out.printf("</%s>\n", structType);
+    }
+
+    /**
+     * @see #showStructure(PDStructureNode, Map)
+     * @see #testExtractTestWPhromma()
+     */
+    void showContent(int mcid, Map<Integer, PDMarkedContent> theseMarkedContents) {
+        PDMarkedContent markedContent = theseMarkedContents != null ? theseMarkedContents.get(mcid) : null;
+        List<Object> contents = markedContent != null ? markedContent.getContents() : Collections.emptyList();
+        StringBuilder textContent =  new StringBuilder();
+        for (Object object : contents) {
+            if (object instanceof TextPosition) {
+                textContent.append(((TextPosition)object).getUnicode());
+            } else {
+                textContent.append("?" + object);
+            }
+        }
+        System.out.printf("%s\n", textContent);
+    }
+}
--- a/Show More
+++ b/Show More