Merge branch 'RM-231-bp' into 'release/0.192.x'

RM-231: missing whitespace in name

See merge request fforesight/layout-parser!266

.gitattributes

@@ -0,0 +1 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -18,6 +18,7 @@ target/
 .settings
 .springBeans
 .sts4-cache
+.gradle
 
 ### IntelliJ IDEA ###
 .idea
@@ -37,3 +38,9 @@ build/
 
 ### VS Code ###
 .vscode/
+gradlew.bat
+gradlew
+gradle.properties
+gradle/
+.DS_Store
+.DS_Store/

.gitlab-ci.yml

@@ -1,4 +1,26 @@
+variables:
+  #  SONAR_PROJECT_KEY: 'fforesight_layout-parser_AYd5quv2mRkBOCG22hvF'
+  GIT_SUBMODULE_STRATEGY: recursive
+  GIT_SUBMODULE_FORCE_HTTPS: 'true'
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'
-    file: 'ci-templates/maven_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^feature/ && $CI_COMMIT_TAG == ""
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG

.gitmodules

@@ -0,0 +1,8 @@
+[submodule "layoutparser-service/layoutparser-service-server/src/test/resources/files/basf"]
+	path = layoutparser-service/layoutparser-service-server/src/test/resources/files/basf
+	url = ssh://git@git.knecon.com:22222/fforesight/documents/basf.git
+	update = merge
+[submodule "layoutparser-service/layoutparser-service-server/src/test/resources/files/syngenta"]
+	path = layoutparser-service/layoutparser-service-server/src/test/resources/files/syngenta
+	url = ssh://git@git.knecon.com:22222/fforesight/documents/syngenta.git
+	update = merge

.mvn/wrapper/maven-wrapper.properties

@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.7/apache-maven-3.8.7-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar

README.md

@@ -1 +1,89 @@
+# PDF Layout Parser Micro-Service: layout-parser  
 
+## Introduction
+The layout-parser micro-service is a powerful tool designed to efficiently extract structured information from PDF documents. Written in Java and utilizing Spring Boot 3, Apache PDFBox, and RabbitMQ, this micro-service excels at parsing PDFs and organizing their content into a meaningful and coherent layout structure. Notably, the layout-parser micro-service distinguishes itself by relying solely on advanced algorithms, rather than machine learning techniques.
+
+### Key Steps in the PDF Layout Parsing Process:
+
+* **Text Position Extraction:**  
+The micro-service leverages Apache PDFBox to extract precise text positions for each individual character within the PDF document.
+
+* **Word Segmentation and Text Block Formation:**  
+Employing an array of diverse algorithms, the micro-service initially identifies and segments words, creating distinct text blocks.
+
+* **Text Block Classification:**  
+The segmented text blocks are then subjected to classification algorithms. These algorithms categorize the text blocks based on their content and visual properties, distinguishing between sections, subsections, headlines, paragraphs, images, tables, table cells, headers, and footers.
+
+* **Layout Coherence Establishment:**  
+The classified text blocks are subsequently orchestrated into a cohesive layout structure. This process involves arranging sections, subsections, paragraphs, images, and other elements in a logical and structured manner.
+
+* **Output Generation in Various Formats:**  
+Once the layout structure is established, the micro-service generates output in multiple formats. These formats are designed for seamless integration with downstream micro-services. The supported formats include JSON, XML, and others, ensuring flexibility in downstream data consumption.
+
+### Optional Enhancements:
+
+* **ML-Based Table Extraction:**  
+For enhanced results, users have the option to incorporate machine learning-based table extraction. This feature can be activated by providing ML-generated results as a JSON file, which are then integrated seamlessly into the layout structure.
+
+* **Image Classification using ML:**  
+Additionally, for more accurate image classification, users can optionally feed ML-generated image classification results into the micro-service. Similar to the table extraction option, the micro-service processes the pre-parsed results in JSON format, thus optimizing the accuracy of image content identification.
+
+In conclusion, the layout-parser micro-service is a versatile PDF layout parsing solution crafted entirely around advanced algorithms, without reliance on machine learning. It proficiently extracts text positions, segments content into meaningful blocks, classifies these blocks, arranges them coherently, and outputs structured data for downstream micro-services. Optional integration with ML-generated table extractions and image classifications further enhances its capabilities.
+
+
+
+
+
+
+## Installation
+
+### Prerequisites
+
+Before building and using the layout-parser micro-service, please ensure you have the following software and tools installed:
+
+Java Development Kit (JDK) 17 or later
+Gradle build tool (preinstalled)
+Build and Test
+To build and test the micro-service, follow these steps:
+
+### Clone the Repository:
+
+bash
+```
+git clone ssh://git@git.knecon.com:22222/fforesight/layout-parser.git
+cd layout-parser
+```
+### Build the Project:
+Use the following command to build the project using Gradle:
+
+```
+gradle clean build
+```
+### Run Tests:
+Run the test suite using the following command:
+```
+gradle test
+```
+## Building a Custom Docker Image
+To create a custom Docker image for the layout-parser micro-service, execute the provided script:
+
+### Ensure Docker is Installed:
+Ensure that Docker is installed and running on your system.
+
+### Run the Image Building Script:
+Execute the publish-custom-image script in the project directory:
+
+```
+./publish-custom-image
+```
+## Publishing to Internal Maven Repository
+To publish the layout-parser micro-service to your internal Maven repository, execute the following command:
+
+```
+gradle -Pversion=buildVersion publish
+```
+Replace buildVersion with the desired version number.
+
+## Additional Notes
+Make sure to configure any necessary application properties before deploying the micro-service.
+For advanced usage and configurations, refer to Kilian or Dom or preferably the source code.

buildSrc/build.gradle.kts

@@ -0,0 +1,7 @@
+plugins {
+    `kotlin-dsl`
+}
+
+repositories {
+    gradlePluginPortal()
+}

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -0,0 +1,91 @@
+plugins {
+    `java-library`
+    `maven-publish`
+    pmd
+    checkstyle
+    jacoco
+}
+
+group = "com.knecon.fforesight"
+
+java.sourceCompatibility = JavaVersion.VERSION_17
+java.targetCompatibility = JavaVersion.VERSION_17
+
+tasks.pmdMain {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
+}
+
+tasks.pmdTest {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/test_pmd.xml")
+}
+
+tasks.named<Test>("test") {
+    useJUnitPlatform()
+    reports {
+        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
+    }
+    minHeapSize = "512m"
+    maxHeapSize = "2048m"
+}
+
+tasks.test {
+    finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
+}
+
+tasks.jacocoTestReport {
+    dependsOn(tasks.test) // tests are required to run before generating the report
+    reports {
+        xml.required.set(true)
+        csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
+    }
+}
+
+allprojects {
+
+    tasks.withType<Javadoc> {
+        options {
+            this as StandardJavadocDocletOptions
+            addBooleanOption("Xdoclint:none", true)
+            addStringOption("Xmaxwarns", "1")
+        }
+    }
+
+    pmd {
+        setConsoleOutput(true)
+    }
+
+    publishing {
+        publications {
+            create<MavenPublication>(name) {
+                from(components["java"])
+            }
+        }
+        repositories {
+            maven {
+                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+                credentials {
+                    username = providers.gradleProperty("mavenUser").getOrNull();
+                    password = providers.gradleProperty("mavenPassword").getOrNull();
+                }
+            }
+        }
+    }
+}
+
+java {
+    withJavadocJar()
+}
+
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven {
+        url = uri("https://nexus.knecon.com/repository/gindev/")
+        credentials {
+            username = providers.gradleProperty("mavenUser").getOrNull();
+            password = providers.gradleProperty("mavenPassword").getOrNull();
+        }
+    }
+}

config/checkstyle/checkstyle.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <property
+            name="severity"
+            value="error"/>
+    <module name="TreeWalker">
+        <module name="SuppressWarningsHolder"/>
+        <module name="MissingDeprecated"/>
+        <module name="MissingOverride"/>
+        <module name="AnnotationLocation"/>
+        <module name="JavadocStyle"/>
+        <module name="NonEmptyAtclauseDescription"/>
+        <module name="IllegalImport"/>
+        <module name="RedundantImport"/>
+        <module name="RedundantModifier"/>
+        <module name="EmptyBlock"/>
+        <module name="DefaultComesLast"/>
+        <module name="EmptyStatement"/>
+        <module name="EqualsHashCode"/>
+        <module name="ExplicitInitialization"/>
+        <module name="IllegalInstantiation"/>
+        <module name="ModifiedControlVariable"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="PackageDeclaration"/>
+        <module name="ParameterAssignment"/>
+        <module name="SimplifyBooleanExpression"/>
+        <module name="SimplifyBooleanReturn"/>
+        <module name="StringLiteralEquality"/>
+        <module name="OneStatementPerLine"/>
+        <module name="FinalClass"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="UpperEll"/>
+        <module name="OuterTypeFilename"/>
+    </module>
+    <module name="FileTabCharacter"/>
+    <module name="SuppressWarningsFilter"/>
+</module>

config/pmd/pmd.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon ruleset checks the code for bad stuff
+    </description>
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AssignmentInOperand"/>
+        <exclude name="BeanMembersShouldSerialize"/>
+    </rule>
+
+</ruleset>
+

config/pmd/test_pmd.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon test ruleset checks the code for bad stuff
+    </description>
+
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AssignmentInOperand"/>
+        <exclude name="TestClassWithoutTestCases"/>
+        <exclude name="BeanMembersShouldSerialize"/>
+    </rule>
+
+</ruleset>

gradle.properties.kts

@@ -0,0 +1 @@
+version = 0.1-SNAPSHOT

layoutparser-service-image/pom.xml

@@ -1,99 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>platform-docker-dependency</artifactId>
-        <version>0.1.0</version>
-        <relativePath/>
-    </parent>
-
-    <groupId>com.knecon.fforesight</groupId>
-    <artifactId>layoutparser-service-image</artifactId>
-    <version>0.1-SNAPSHOT</version>
-    <packaging>pom</packaging>
-
-    <properties>
-        <service.server>layoutparser-service-server</service.server>
-        <platform.jar>${service.server}.jar</platform.jar>
-        <docker.skip.push>false</docker.skip.push>
-        <docker.image.prefix>ff</docker.image.prefix>
-        <docker.image.name>${docker.image.prefix}/${service.server}</docker.image.name>
-    </properties>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-dependency-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-resources-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>io.fabric8</groupId>
-                <artifactId>docker-maven-plugin</artifactId>
-            </plugin>
-        </plugins>
-
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-dependency-plugin</artifactId>
-                    <executions>
-                        <execution>
-                            <id>download-platform-jar</id>
-                            <phase>prepare-package</phase>
-                            <goals>
-                                <goal>copy</goal>
-                            </goals>
-                            <configuration>
-                                <artifactItems>
-                                    <dependency>
-                                        <groupId>${project.groupId}</groupId>
-                                        <artifactId>${service.server}</artifactId>
-                                        <version>${project.version}</version>
-                                        <type>jar</type>
-                                        <overWrite>true</overWrite>
-                                        <destFileName>${platform.jar}</destFileName>
-                                    </dependency>
-                                </artifactItems>
-                                <outputDirectory>${docker.build.directory}</outputDirectory>
-                            </configuration>
-                        </execution>
-                    </executions>
-                </plugin>
-                <plugin>
-                    <groupId>io.fabric8</groupId>
-                    <artifactId>docker-maven-plugin</artifactId>
-                    <configuration>
-                        <images>
-                            <image>
-                                <name>${docker.image.name}</name>
-                                <build>
-                                    <dockerFileDir>${docker.build.directory}</dockerFileDir>
-                                    <args>
-                                        <PLATFORM_JAR>${platform.jar}</PLATFORM_JAR>
-                                    </args>
-                                    <tags>
-                                        <tag>${docker.image.version}</tag>
-                                        <tag>latest</tag>
-                                    </tags>
-                                </build>
-                            </image>
-                        </images>
-                    </configuration>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
-</project>

layoutparser-service-image/src/main/docker/Dockerfile

@@ -1,9 +0,0 @@
-FROM red/base-image:2.0.2
-
-ARG PLATFORM_JAR
-
-ENV PLATFORM_JAR ${PLATFORM_JAR}
-
-ENV USES_ELASTICSEARCH false
-
-COPY ["${PLATFORM_JAR}", "/"]

layoutparser-service/layoutparser-service-internal-api/build.gradle.kts

@@ -0,0 +1,11 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.4"
+}
+
+description = "layoutparser-service-internal-api"
+
+dependencies {
+    implementation("io.swagger.core.v3:swagger-annotations:2.2.15")
+    implementation("com.google.protobuf:protobuf-java-util:4.27.1")
+}

layoutparser-service/layoutparser-service-internal-api/pom.xml

@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>0.1-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>layoutparser-service-internal-api</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>${guava.version}</version>
-        </dependency>
-    </dependencies>
-
-</project>

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentData.java

@@ -1,5 +1,13 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.io.Serializable;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
+
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -10,12 +18,26 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class DocumentData {
+@Schema(description = "Object containing the complete document layout parsing information. It is split into 4 categories, structure, text, positions and pages: "
+                      + "The document tree structure of SemanticNodes such as Section, Paragraph, Headline, etc. "
+                      + "The text, which is stored as separate blocks of data. "
+                      + "The text positions, which are also stored as separate blocks. The Blocks are equal to the text blocks in length and order. "
+                      + "The page information.")
+public class DocumentData implements Serializable {
 
-    DocumentPage[] documentPages;
-    DocumentTextData[] documentTextData;
-    DocumentPositionData[] documentPositions;
-    DocumentStructure documentStructure;
+    @Schema(description = "Contains information about the document's pages.")
+    AllDocumentPages documentPages;
+    @Schema(description = "Contains information about the document's text.")
+    AllDocumentTextData documentTextData;
+    @Schema(description = "Contains information about the document's text positions.")
+    AllDocumentPositionData documentPositions;
+    @Schema(description = "Contains information about the document's semantic structure.")
+    DocumentStructureWrapper documentStructureWrapper;
 
 
+    public DocumentStructure getDocumentStructure() {
+
+        return documentStructureWrapper.getDocumentStructure();
+    }
+
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPage.java

@@ -1,20 +1,31 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Deprecated
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class DocumentPage {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@Schema(description = "Object containing information about the document's pages.")
+public class DocumentPage implements Serializable {
 
+    @Schema(description = "The page number, starting with 1.")
     int number;
+    @Schema(description = "The page height in PDF user units.", example = "792")
     int height;
+    @Schema(description = "The page width in PDF user units.", example = "694")
     int width;
+    @Schema(description = "The page rotation as specified by the PDF.", example = "90", allowableValues = {"0", "90", "180", "270"})
     int rotation;
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPositionData.java

@@ -1,19 +1,29 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Deprecated
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class DocumentPositionData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@Schema(description = "Object containing text positional information of a specific text block. A document is split into multiple text blocks, which are supposed to be read in order. Every text block can only occur on a single page.")
+public class DocumentPositionData implements Serializable {
 
+    @Schema(description = "Identifier of the text block.")
     Long id;
+    @Schema(description = "For each string coordinate in the search text of the text block, the array contains an entry relating the string coordinate to the position coordinate. This is required due to the text and position coordinates not being equal.")
     int[] stringIdxToPositionIdx;
+    @Schema(description = "The bounding box for each glyph as a rectangle. This matrix is of size (n,4), where n is the number of glyphs in the text block. The second dimension specifies the rectangle with the value x, y, width, height, with x, y specifying the lower left corner. In order to access this information, the stringIdxToPositionIdx array must be used to transform the coordinates.")
     float[][] positions;
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -1,26 +1,93 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.awt.geom.Rectangle2D;
+import java.io.Serializable;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Stream;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Deprecated
 @Data
 @Builder
-@AllArgsConstructor
 @NoArgsConstructor
+@AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentStructure {
+@Schema(description = "Object containing information about the parsed tree structure of the SemanticNodes, such as Section, Paragraph, Headline etc inside of the document.")
+public class DocumentStructure implements Serializable {
 
+    @Schema(description = "The root EntryData represents the Document.")
     EntryData root;
 
+    @Schema(description = "Object containing the extra field names, a table has in its properties field.")
+    public static class TableProperties implements Serializable {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, an Image has in its properties field.")
+    public static class ImageProperties implements Serializable {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+        public static final String REPRESENTATION_HASH = "representationHash";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, a table cell has in its properties field.")
+    public static class TableCellProperties implements Serializable {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, a duplicate paragraph has in its properties field.")
+    public static class DuplicateParagraphProperties implements Serializable {
+
+        public static final String UNSORTED_TEXTBLOCK_ID = "utbid";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER))
+                .map(Float::parseFloat)
+                .toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
+    public static double[] parseRepresentationVector(String representationHash) {
+
+        String[] stringArray = representationHash.split("[,\\s]+");
+        double[] doubleArray = new double[stringArray.length];
+        for (int i = 0; i < stringArray.length; i++) {
+            doubleArray[i] = Double.parseDouble(stringArray[i]);
+        }
+
+        return doubleArray;
+    }
+
 
     public EntryData get(List<Integer> tocId) {
 
@@ -37,34 +104,50 @@ public class DocumentStructure {
 
     public Stream<EntryData> streamAllEntries() {
 
-        return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentStructure::flatten);
+        return Stream.concat(Stream.of(root), root.children.stream())
+                .flatMap(DocumentStructure::flatten);
     }
 
 
     public String toString() {
 
-        return String.join("\n", streamAllEntries().map(EntryData::toString).toList());
+        return String.join("\n",
+                           streamAllEntries().map(EntryData::toString)
+                                   .toList());
     }
 
 
     private static Stream<EntryData> flatten(EntryData entry) {
 
-        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentStructure::flatten));
+        return Stream.concat(Stream.of(entry),
+                             entry.children.stream()
+                                     .flatMap(DocumentStructure::flatten));
     }
 
 
+    @Data
     @Builder
-    @Getter
+    @NoArgsConstructor
     @AllArgsConstructor
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    public static class EntryData {
+    @FieldDefaults(level = AccessLevel.PRIVATE)
+    @Schema(description = "Object containing information of a SemanticNode and also structuring the layout with children.")
+    public static class EntryData implements Serializable {
 
+        @Schema(description = "Type of the semantic node.", allowableValues = {"DOCUMENT", "SECTION", "PARAGRAPH", "HEADLINE", "TABLE", "TABLE_CELL", "HEADER", "FOOTER", "IMAGE"})
         NodeType type;
+        @Schema(description = "Specifies the position in the parsed tree structure.", example = "[1, 0, 2]")
         int[] treeId;
+        @Schema(description = "Specifies the text block IDs associated with this semantic node. The value should be joined with the DocumentTextData/DocumentPositionData. Is empty, if no text block is directly associated with this semantic node. Only Paragraph, Headline, Header or Footer is directly associated with a text block.", example = "[1]")
         Long[] atomicBlockIds;
+        @Schema(description = "Specifies the pages this semantic node appears on. The value should be joined with the PageData.", example = "[1, 2, 3]")
         Long[] pageNumbers;
+        @Schema(description = "Some semantic nodes have additional information, this information is stored in this Map. The extra fields are specified by the Properties subclasses.", example = "For a Table: {\"numberOfRows\": 3, \"numberOfCols\": 4}")
         Map<String, String> properties;
+        @Schema(description = "All child Entries of this Entry.", example = "[1, 2, 3]")
         List<EntryData> children;
+        @Schema(description = "Describes the origin of the semantic node", example = "[ALGORITHM]")
+        Set<LayoutEngine> engines;
+
 
         @Override
         public String toString() {

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructureProto.java

@@ -0,0 +1,799 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: DocumentStructure.proto
+// Protobuf Java Version: 4.27.1
+@SuppressWarnings("all")
+public final class DocumentStructureProto {
+
+    private DocumentStructureProto() {}
+
+
+    static {
+        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                /* major= */ 4,
+                /* minor= */ 27,
+                /* patch= */ 1,
+                /* suffix= */ "", DocumentStructureProto.class.getName());
+    }
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
+
+    }
+
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
+
+        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
+    }
+
+
+    public interface DocumentStructureOrBuilder extends
+            // @@protoc_insertion_point(interface_extends:DocumentStructure)
+            com.google.protobuf.MessageOrBuilder {
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         *
+         * @return Whether the root field is set.
+         */
+        boolean hasRoot();
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         *
+         * @return The root.
+         */
+        EntryDataProto.EntryData getRoot();
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         */
+        EntryDataProto.EntryDataOrBuilder getRootOrBuilder();
+
+    }
+
+    /**
+     * Protobuf type {@code DocumentStructure}
+     */
+    public static final class DocumentStructure extends com.google.protobuf.GeneratedMessage implements
+            // @@protoc_insertion_point(message_implements:DocumentStructure)
+            DocumentStructureOrBuilder {
+
+        private static final long serialVersionUID = 0L;
+
+        static {
+            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                    /* major= */ 4,
+                    /* minor= */ 27,
+                    /* patch= */ 1,
+                    /* suffix= */ "", DocumentStructure.class.getName());
+        }
+
+        // Use DocumentStructure.newBuilder() to construct.
+        private DocumentStructure(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+
+            super(builder);
+        }
+
+
+        private DocumentStructure() {
+
+        }
+
+
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+
+            return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+        }
+
+
+        @Override
+        protected FieldAccessorTable internalGetFieldAccessorTable() {
+
+            return DocumentStructureProto.internal_static_DocumentStructure_fieldAccessorTable.ensureFieldAccessorsInitialized(DocumentStructure.class, Builder.class);
+        }
+
+
+        private int bitField0_;
+        public static final int ROOT_FIELD_NUMBER = 1;
+        private EntryDataProto.EntryData root_;
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         *
+         * @return Whether the root field is set.
+         */
+        @Override
+        public boolean hasRoot() {
+
+            return ((bitField0_ & 0x00000001) != 0);
+        }
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         *
+         * @return The root.
+         */
+        @Override
+        public EntryDataProto.EntryData getRoot() {
+
+            return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
+        }
+
+
+        /**
+         * <pre>
+         * The root EntryData represents the Document.
+         * </pre>
+         *
+         * <code>.EntryData root = 1;</code>
+         */
+        @Override
+        public EntryDataProto.EntryDataOrBuilder getRootOrBuilder() {
+
+            return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
+        }
+
+
+        private byte memoizedIsInitialized = -1;
+
+
+        @Override
+        public final boolean isInitialized() {
+
+            byte isInitialized = memoizedIsInitialized;
+          if (isInitialized == 1) {
+            return true;
+          }
+          if (isInitialized == 0) {
+            return false;
+          }
+
+            memoizedIsInitialized = 1;
+            return true;
+        }
+
+
+        @Override
+        public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+
+            if (((bitField0_ & 0x00000001) != 0)) {
+                output.writeMessage(1, getRoot());
+            }
+            getUnknownFields().writeTo(output);
+        }
+
+
+        @Override
+        public int getSerializedSize() {
+
+            int size = memoizedSize;
+          if (size != -1) {
+            return size;
+          }
+
+            size = 0;
+            if (((bitField0_ & 0x00000001) != 0)) {
+                size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, getRoot());
+            }
+            size += getUnknownFields().getSerializedSize();
+            memoizedSize = size;
+            return size;
+        }
+
+
+        @Override
+        public boolean equals(final Object obj) {
+
+            if (obj == this) {
+                return true;
+            }
+            if (!(obj instanceof DocumentStructure)) {
+                return super.equals(obj);
+            }
+            DocumentStructure other = (DocumentStructure) obj;
+
+          if (hasRoot() != other.hasRoot()) {
+            return false;
+          }
+            if (hasRoot()) {
+              if (!getRoot().equals(other.getRoot())) {
+                return false;
+              }
+            }
+          if (!getUnknownFields().equals(other.getUnknownFields())) {
+            return false;
+          }
+            return true;
+        }
+
+
+        @Override
+        public int hashCode() {
+
+            if (memoizedHashCode != 0) {
+                return memoizedHashCode;
+            }
+            int hash = 41;
+            hash = (19 * hash) + getDescriptor().hashCode();
+            if (hasRoot()) {
+                hash = (37 * hash) + ROOT_FIELD_NUMBER;
+                hash = (53 * hash) + getRoot().hashCode();
+            }
+            hash = (29 * hash) + getUnknownFields().hashCode();
+            memoizedHashCode = hash;
+            return hash;
+        }
+
+
+        public static DocumentStructure parseFrom(java.nio.ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data);
+        }
+
+
+        public static DocumentStructure parseFrom(java.nio.ByteBuffer data,
+                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseFrom(com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data);
+        }
+
+
+        public static DocumentStructure parseFrom(com.google.protobuf.ByteString data,
+                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data);
+        }
+
+
+        public static DocumentStructure parseFrom(byte[] data,
+                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
+
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseFrom(java.io.InputStream input) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input);
+        }
+
+
+        public static DocumentStructure parseFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseDelimitedWithIOException(PARSER, input);
+        }
+
+
+        public static DocumentStructure parseDelimitedFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseDelimitedWithIOException(PARSER, input, extensionRegistry);
+        }
+
+
+        public static DocumentStructure parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input);
+        }
+
+
+        public static DocumentStructure parseFrom(com.google.protobuf.CodedInputStream input,
+                                                  com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
+
+            return com.google.protobuf.GeneratedMessage.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+
+        @Override
+        public Builder newBuilderForType() {return newBuilder();}
+
+
+        public static Builder newBuilder() {
+
+            return DEFAULT_INSTANCE.toBuilder();
+        }
+
+
+        public static Builder newBuilder(DocumentStructure prototype) {
+
+            return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+        }
+
+
+        @Override
+        public Builder toBuilder() {
+
+            return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+        }
+
+
+        @Override
+        protected Builder newBuilderForType(BuilderParent parent) {
+
+            Builder builder = new Builder(parent);
+            return builder;
+        }
+
+
+        /**
+         * Protobuf type {@code DocumentStructure}
+         */
+        public static final class Builder extends com.google.protobuf.GeneratedMessage.Builder<Builder> implements
+                // @@protoc_insertion_point(builder_implements:DocumentStructure)
+                DocumentStructureOrBuilder {
+
+            public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+
+                return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+            }
+
+
+            @Override
+            protected FieldAccessorTable internalGetFieldAccessorTable() {
+
+                return DocumentStructureProto.internal_static_DocumentStructure_fieldAccessorTable.ensureFieldAccessorsInitialized(DocumentStructure.class, Builder.class);
+            }
+
+
+            // Construct using DocumentStructureOuterClass.DocumentStructure.newBuilder()
+            private Builder() {
+
+                maybeForceBuilderInitialization();
+            }
+
+
+            private Builder(BuilderParent parent) {
+
+                super(parent);
+                maybeForceBuilderInitialization();
+            }
+
+
+            private void maybeForceBuilderInitialization() {
+
+                if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+                    getRootFieldBuilder();
+                }
+            }
+
+
+            @Override
+            public Builder clear() {
+
+                super.clear();
+                bitField0_ = 0;
+                root_ = null;
+                if (rootBuilder_ != null) {
+                    rootBuilder_.dispose();
+                    rootBuilder_ = null;
+                }
+                return this;
+            }
+
+
+            @Override
+            public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+
+                return DocumentStructureProto.internal_static_DocumentStructure_descriptor;
+            }
+
+
+            @Override
+            public DocumentStructure getDefaultInstanceForType() {
+
+                return DocumentStructure.getDefaultInstance();
+            }
+
+
+            @Override
+            public DocumentStructure build() {
+
+                DocumentStructure result = buildPartial();
+                if (!result.isInitialized()) {
+                    throw newUninitializedMessageException(result);
+                }
+                return result;
+            }
+
+
+            @Override
+            public DocumentStructure buildPartial() {
+
+                DocumentStructure result = new DocumentStructure(this);
+                if (bitField0_ != 0) {
+                    buildPartial0(result);
+                }
+                onBuilt();
+                return result;
+            }
+
+
+            private void buildPartial0(DocumentStructure result) {
+
+                int from_bitField0_ = bitField0_;
+                int to_bitField0_ = 0;
+                if (((from_bitField0_ & 0x00000001) != 0)) {
+                    result.root_ = rootBuilder_ == null ? root_ : rootBuilder_.build();
+                    to_bitField0_ |= 0x00000001;
+                }
+                result.bitField0_ |= to_bitField0_;
+            }
+
+
+            @Override
+            public Builder mergeFrom(com.google.protobuf.Message other) {
+
+                if (other instanceof DocumentStructure) {
+                    return mergeFrom((DocumentStructure) other);
+                } else {
+                    super.mergeFrom(other);
+                    return this;
+                }
+            }
+
+
+            public Builder mergeFrom(DocumentStructure other) {
+
+              if (other == DocumentStructure.getDefaultInstance()) {
+                return this;
+              }
+                if (other.hasRoot()) {
+                    mergeRoot(other.getRoot());
+                }
+                this.mergeUnknownFields(other.getUnknownFields());
+                onChanged();
+                return this;
+            }
+
+
+            @Override
+            public final boolean isInitialized() {
+
+                return true;
+            }
+
+
+            @Override
+            public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
+
+                if (extensionRegistry == null) {
+                    throw new NullPointerException();
+                }
+                try {
+                    boolean done = false;
+                    while (!done) {
+                        int tag = input.readTag();
+                        switch (tag) {
+                            case 0:
+                                done = true;
+                                break;
+                            case 10: {
+                                input.readMessage(getRootFieldBuilder().getBuilder(), extensionRegistry);
+                                bitField0_ |= 0x00000001;
+                                break;
+                            } // case 10
+                            default: {
+                                if (!super.parseUnknownField(input, extensionRegistry, tag)) {
+                                    done = true; // was an endgroup tag
+                                }
+                                break;
+                            } // default:
+                        } // switch (tag)
+                    } // while (!done)
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                    throw e.unwrapIOException();
+                } finally {
+                    onChanged();
+                } // finally
+                return this;
+            }
+
+
+            private int bitField0_;
+
+            private EntryDataProto.EntryData root_;
+            private com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder> rootBuilder_;
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             *
+             * @return Whether the root field is set.
+             */
+            public boolean hasRoot() {
+
+                return ((bitField0_ & 0x00000001) != 0);
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             *
+             * @return The root.
+             */
+            public EntryDataProto.EntryData getRoot() {
+
+                if (rootBuilder_ == null) {
+                    return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
+                } else {
+                    return rootBuilder_.getMessage();
+                }
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public Builder setRoot(EntryDataProto.EntryData value) {
+
+                if (rootBuilder_ == null) {
+                    if (value == null) {
+                        throw new NullPointerException();
+                    }
+                    root_ = value;
+                } else {
+                    rootBuilder_.setMessage(value);
+                }
+                bitField0_ |= 0x00000001;
+                onChanged();
+                return this;
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public Builder setRoot(EntryDataProto.EntryData.Builder builderForValue) {
+
+                if (rootBuilder_ == null) {
+                    root_ = builderForValue.build();
+                } else {
+                    rootBuilder_.setMessage(builderForValue.build());
+                }
+                bitField0_ |= 0x00000001;
+                onChanged();
+                return this;
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public Builder mergeRoot(EntryDataProto.EntryData value) {
+
+                if (rootBuilder_ == null) {
+                    if (((bitField0_ & 0x00000001) != 0) && root_ != null && root_ != EntryDataProto.EntryData.getDefaultInstance()) {
+                        getRootBuilder().mergeFrom(value);
+                    } else {
+                        root_ = value;
+                    }
+                } else {
+                    rootBuilder_.mergeFrom(value);
+                }
+                if (root_ != null) {
+                    bitField0_ |= 0x00000001;
+                    onChanged();
+                }
+                return this;
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public Builder clearRoot() {
+
+                bitField0_ = (bitField0_ & ~0x00000001);
+                root_ = null;
+                if (rootBuilder_ != null) {
+                    rootBuilder_.dispose();
+                    rootBuilder_ = null;
+                }
+                onChanged();
+                return this;
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public EntryDataProto.EntryData.Builder getRootBuilder() {
+
+                bitField0_ |= 0x00000001;
+                onChanged();
+                return getRootFieldBuilder().getBuilder();
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            public EntryDataProto.EntryDataOrBuilder getRootOrBuilder() {
+
+                if (rootBuilder_ != null) {
+                    return rootBuilder_.getMessageOrBuilder();
+                } else {
+                    return root_ == null ? EntryDataProto.EntryData.getDefaultInstance() : root_;
+                }
+            }
+
+
+            /**
+             * <pre>
+             * The root EntryData represents the Document.
+             * </pre>
+             *
+             * <code>.EntryData root = 1;</code>
+             */
+            private com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder> getRootFieldBuilder() {
+
+                if (rootBuilder_ == null) {
+                    rootBuilder_ = new com.google.protobuf.SingleFieldBuilder<EntryDataProto.EntryData, EntryDataProto.EntryData.Builder, EntryDataProto.EntryDataOrBuilder>(getRoot(),
+                                                                                                                                                                             getParentForChildren(),
+                                                                                                                                                                             isClean());
+                    root_ = null;
+                }
+                return rootBuilder_;
+            }
+
+            // @@protoc_insertion_point(builder_scope:DocumentStructure)
+        }
+
+        // @@protoc_insertion_point(class_scope:DocumentStructure)
+        private static final DocumentStructure DEFAULT_INSTANCE;
+
+        static {
+            DEFAULT_INSTANCE = new DocumentStructure();
+        }
+
+        public static DocumentStructure getDefaultInstance() {
+
+            return DEFAULT_INSTANCE;
+        }
+
+
+        private static final com.google.protobuf.Parser<DocumentStructure> PARSER = new com.google.protobuf.AbstractParser<DocumentStructure>() {
+            @Override
+            public DocumentStructure parsePartialFrom(com.google.protobuf.CodedInputStream input,
+                                                      com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
+
+                Builder builder = newBuilder();
+                try {
+                    builder.mergeFrom(input, extensionRegistry);
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                    throw e.setUnfinishedMessage(builder.buildPartial());
+                } catch (com.google.protobuf.UninitializedMessageException e) {
+                    throw e.asInvalidProtocolBufferException().setUnfinishedMessage(builder.buildPartial());
+                } catch (java.io.IOException e) {
+                    throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(builder.buildPartial());
+                }
+                return builder.buildPartial();
+            }
+        };
+
+
+        public static com.google.protobuf.Parser<DocumentStructure> parser() {
+
+            return PARSER;
+        }
+
+
+        @Override
+        public com.google.protobuf.Parser<DocumentStructure> getParserForType() {
+
+            return PARSER;
+        }
+
+
+        @Override
+        public DocumentStructure getDefaultInstanceForType() {
+
+            return DEFAULT_INSTANCE;
+        }
+
+    }
+
+    private static final com.google.protobuf.Descriptors.Descriptor internal_static_DocumentStructure_descriptor;
+    private static final com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_DocumentStructure_fieldAccessorTable;
+
+
+    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
+
+        return descriptor;
+    }
+
+
+    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
+
+    static {
+        String[] descriptorData = {"\n\027DocumentStructure.proto\032\017EntryData.pro"
+                                   + "to\"-\n\021DocumentStructure\022\030\n\004root\030\001 \001(\0132\n."
+                                   + "EntryDatab\006proto3"};
+        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData,
+                                                                                                   new com.google.protobuf.Descriptors.FileDescriptor[]{EntryDataProto.getDescriptor(),});
+        internal_static_DocumentStructure_descriptor = getDescriptor().getMessageTypes()
+                .get(0);
+        internal_static_DocumentStructure_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable(internal_static_DocumentStructure_descriptor,
+                                                                                                                           new String[]{"Root",});
+        descriptor.resolveAllFeaturesImmutable();
+        EntryDataProto.getDescriptor();
+    }
+
+    // @@protoc_insertion_point(outer_class_scope)
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructureWrapper.java

@@ -0,0 +1,126 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
+
+import java.awt.geom.Rectangle2D;
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@Getter
+@AllArgsConstructor
+public class DocumentStructureWrapper implements Serializable {
+
+    private final DocumentStructure documentStructure;
+
+
+    @Schema(description = "Object containing the extra field names, a table has in its properties field.")
+    public static class TableProperties implements Serializable {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, an Image has in its properties field.")
+    public static class ImageProperties implements Serializable {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+        public static final String REPRESENTATION_HASH = "representationHash";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, a table cell has in its properties field.")
+    public static class TableCellProperties implements Serializable {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, a duplicate paragraph has in its properties field.")
+    public static class DuplicateParagraphProperties implements Serializable {
+
+        public static final String UNSORTED_TEXTBLOCK_ID = "utbid";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER))
+                .map(Float::parseFloat)
+                .toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
+    public static double[] parseRepresentationVector(String representationHash) {
+
+        String[] stringArray = representationHash.split("[,\\s]+");
+        double[] doubleArray = new double[stringArray.length];
+        for (int i = 0; i < stringArray.length; i++) {
+            doubleArray[i] = Double.parseDouble(stringArray[i]);
+        }
+
+        return doubleArray;
+    }
+
+
+    public EntryData get(List<Integer> tocId) {
+
+        if (tocId.isEmpty()) {
+            return documentStructure.getRoot();
+        }
+        EntryData entry = documentStructure.getRoot().getChildrenList()
+                .get(tocId.get(0));
+        for (int id : tocId.subList(1, tocId.size())) {
+            entry = entry.getChildrenList()
+                    .get(id);
+        }
+        return entry;
+    }
+
+
+    public Stream<EntryData> streamAllEntries() {
+
+        return Stream.concat(Stream.of(documentStructure.getRoot()),
+                             documentStructure.getRoot().getChildrenList()
+                                     .stream())
+                .flatMap(DocumentStructureWrapper::flatten);
+    }
+
+
+    public String toString() {
+
+        return String.join("\n",
+                           streamAllEntries().map(EntryData::toString)
+                                   .toList());
+    }
+
+
+    private static Stream<EntryData> flatten(EntryData entry) {
+
+        return Stream.concat(Stream.of(entry),
+                             entry.getChildrenList()
+                                     .stream()
+                                     .flatMap(DocumentStructureWrapper::flatten));
+    }
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentTextData.java

@@ -1,27 +1,37 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.io.Serializable;
 
-
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Deprecated
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class DocumentTextData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@Schema(description = "Object containing text information of a specific text block. A document is split into multiple text blocks, which are supposed to be read in order. Every text block can only occur on a single page.")
+public class DocumentTextData implements Serializable {
 
+    @Schema(description = "Identifier of the text block.")
     Long id;
+    @Schema(description = "The page the text block occurs on.")
     Long page;
+    @Schema(description = "The text the text block.")
     String searchText;
+    @Schema(description = "Each text block is assigned a number on a page, starting from 0.")
     int numberOnPage;
+    @Schema(description = "The text blocks are ordered, this number represents the start of the text block as a string offset.")
     int start;
+    @Schema(description = "The text blocks are ordered, this number represents the end of the text block as a string offset.")
     int end;
+    @Schema(description = "The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.", example = "[5, 10]")
     int[] lineBreaks;
 
-
-
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+@Deprecated
+public enum LayoutEngine {
+    ALGORITHM,
+    AI,
+    OUTLINE
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngineProto.java

@@ -0,0 +1,193 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: LayoutEngine.proto
+// Protobuf Java Version: 4.27.1
+@SuppressWarnings("all")
+public final class LayoutEngineProto {
+
+    private LayoutEngineProto() {}
+
+
+    static {
+        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                /* major= */ 4,
+                /* minor= */ 27,
+                /* patch= */ 1,
+                /* suffix= */ "", LayoutEngineProto.class.getName());
+    }
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
+
+    }
+
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
+
+        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
+    }
+
+
+    /**
+     * Protobuf enum {@code LayoutEngine}
+     */
+    public enum LayoutEngine implements com.google.protobuf.ProtocolMessageEnum {
+        /**
+         * <code>ALGORITHM = 0;</code>
+         */
+        ALGORITHM(0),
+        /**
+         * <code>AI = 1;</code>
+         */
+        AI(1),
+        /**
+         * <code>OUTLINE = 2;</code>
+         */
+        OUTLINE(2),
+        UNRECOGNIZED(-1),
+        ;
+
+
+        static {
+            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                    /* major= */ 4,
+                    /* minor= */ 27,
+                    /* patch= */ 1,
+                    /* suffix= */ "", LayoutEngine.class.getName());
+        }
+
+        /**
+         * <code>ALGORITHM = 0;</code>
+         */
+        public static final int ALGORITHM_VALUE = 0;
+        /**
+         * <code>AI = 1;</code>
+         */
+        public static final int AI_VALUE = 1;
+        /**
+         * <code>OUTLINE = 2;</code>
+         */
+        public static final int OUTLINE_VALUE = 2;
+
+
+        public final int getNumber() {
+
+            if (this == UNRECOGNIZED) {
+                throw new IllegalArgumentException("Can't get the number of an unknown enum value.");
+            }
+            return value;
+        }
+
+
+        /**
+         * @param value The numeric wire value of the corresponding enum entry.
+         * @return The enum associated with the given numeric wire value.
+         * @deprecated Use {@link #forNumber(int)} instead.
+         */
+        @Deprecated
+        public static LayoutEngine valueOf(int value) {
+
+            return forNumber(value);
+        }
+
+
+        /**
+         * @param value The numeric wire value of the corresponding enum entry.
+         * @return The enum associated with the given numeric wire value.
+         */
+        public static LayoutEngine forNumber(int value) {
+
+            switch (value) {
+                case 0:
+                    return ALGORITHM;
+                case 1:
+                    return AI;
+                case 2:
+                    return OUTLINE;
+                default:
+                    return null;
+            }
+        }
+
+
+        public static com.google.protobuf.Internal.EnumLiteMap<LayoutEngine> internalGetValueMap() {
+
+            return internalValueMap;
+        }
+
+
+        private static final com.google.protobuf.Internal.EnumLiteMap<LayoutEngine> internalValueMap = new com.google.protobuf.Internal.EnumLiteMap<LayoutEngine>() {
+            public LayoutEngine findValueByNumber(int number) {
+
+                return LayoutEngine.forNumber(number);
+            }
+        };
+
+
+        public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() {
+
+            if (this == UNRECOGNIZED) {
+                throw new IllegalStateException("Can't get the descriptor of an unrecognized enum value.");
+            }
+            return getDescriptor().getValues()
+                    .get(ordinal());
+        }
+
+
+        public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() {
+
+            return getDescriptor();
+        }
+
+
+        public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() {
+
+            return LayoutEngineProto.getDescriptor().getEnumTypes()
+                    .get(0);
+        }
+
+
+        private static final LayoutEngine[] VALUES = values();
+
+
+        public static LayoutEngine valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+
+            if (desc.getType() != getDescriptor()) {
+                throw new IllegalArgumentException("EnumValueDescriptor is not for this type.");
+            }
+            if (desc.getIndex() == -1) {
+                return UNRECOGNIZED;
+            }
+            return VALUES[desc.getIndex()];
+        }
+
+
+        private final int value;
+
+
+        private LayoutEngine(int value) {
+
+            this.value = value;
+        }
+
+        // @@protoc_insertion_point(enum_scope:LayoutEngine)
+    }
+
+
+    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
+
+        return descriptor;
+    }
+
+
+    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
+
+    static {
+        String[] descriptorData = {"\n\022LayoutEngine.proto*2\n\014LayoutEngine\022\r\n\t" + "ALGORITHM\020\000\022\006\n\002AI\020\001\022\013\n\007OUTLINE\020\002b\006proto3"};
+        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[]{});
+        descriptor.resolveAllFeaturesImmutable();
+    }
+
+    // @@protoc_insertion_point(outer_class_scope)
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeType.java

@@ -1,10 +1,13 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.io.Serializable;
 import java.util.Locale;
 
-public enum NodeType {
+@Deprecated
+public enum NodeType implements Serializable {
     DOCUMENT,
     SECTION,
+    SUPER_SECTION,
     HEADLINE,
     PARAGRAPH,
     TABLE,

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeTypeProto.java

@@ -0,0 +1,274 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.util.Locale;
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// NO CHECKED-IN PROTOBUF GENCODE
+// source: NodeType.proto
+// Protobuf Java Version: 4.27.1
+@SuppressWarnings("all")
+public final class NodeTypeProto {
+
+    private NodeTypeProto() {}
+
+
+    static {
+        com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                /* major= */ 4,
+                /* minor= */ 27,
+                /* patch= */ 1,
+                /* suffix= */ "", NodeTypeProto.class.getName());
+    }
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
+
+    }
+
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
+
+        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
+    }
+
+
+    /**
+     * Protobuf enum {@code NodeType}
+     */
+    public enum NodeType implements com.google.protobuf.ProtocolMessageEnum {
+        /**
+         * <code>DOCUMENT = 0;</code>
+         */
+        DOCUMENT(0),
+        /**
+         * <code>SECTION = 1;</code>
+         */
+        SECTION(1),
+        /**
+         * <code>SUPER_SECTION = 2;</code>
+         */
+        SUPER_SECTION(2),
+        /**
+         * <code>HEADLINE = 3;</code>
+         */
+        HEADLINE(3),
+        /**
+         * <code>PARAGRAPH = 4;</code>
+         */
+        PARAGRAPH(4),
+        /**
+         * <code>TABLE = 5;</code>
+         */
+        TABLE(5),
+        /**
+         * <code>TABLE_CELL = 6;</code>
+         */
+        TABLE_CELL(6),
+        /**
+         * <code>IMAGE = 7;</code>
+         */
+        IMAGE(7),
+        /**
+         * <code>HEADER = 8;</code>
+         */
+        HEADER(8),
+        /**
+         * <code>FOOTER = 9;</code>
+         */
+        FOOTER(9),
+        UNRECOGNIZED(-1),
+        ;
+
+
+        public String toString() {
+
+            return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
+        }
+
+
+        static {
+            com.google.protobuf.RuntimeVersion.validateProtobufGencodeVersion(com.google.protobuf.RuntimeVersion.RuntimeDomain.PUBLIC,
+                    /* major= */ 4,
+                    /* minor= */ 27,
+                    /* patch= */ 1,
+                    /* suffix= */ "", NodeType.class.getName());
+        }
+
+        /**
+         * <code>DOCUMENT = 0;</code>
+         */
+        public static final int DOCUMENT_VALUE = 0;
+        /**
+         * <code>SECTION = 1;</code>
+         */
+        public static final int SECTION_VALUE = 1;
+        /**
+         * <code>SUPER_SECTION = 2;</code>
+         */
+        public static final int SUPER_SECTION_VALUE = 2;
+        /**
+         * <code>HEADLINE = 3;</code>
+         */
+        public static final int HEADLINE_VALUE = 3;
+        /**
+         * <code>PARAGRAPH = 4;</code>
+         */
+        public static final int PARAGRAPH_VALUE = 4;
+        /**
+         * <code>TABLE = 5;</code>
+         */
+        public static final int TABLE_VALUE = 5;
+        /**
+         * <code>TABLE_CELL = 6;</code>
+         */
+        public static final int TABLE_CELL_VALUE = 6;
+        /**
+         * <code>IMAGE = 7;</code>
+         */
+        public static final int IMAGE_VALUE = 7;
+        /**
+         * <code>HEADER = 8;</code>
+         */
+        public static final int HEADER_VALUE = 8;
+        /**
+         * <code>FOOTER = 9;</code>
+         */
+        public static final int FOOTER_VALUE = 9;
+
+
+        public final int getNumber() {
+
+            if (this == UNRECOGNIZED) {
+                throw new IllegalArgumentException("Can't get the number of an unknown enum value.");
+            }
+            return value;
+        }
+
+
+        /**
+         * @param value The numeric wire value of the corresponding enum entry.
+         * @return The enum associated with the given numeric wire value.
+         * @deprecated Use {@link #forNumber(int)} instead.
+         */
+        @Deprecated
+        public static NodeType valueOf(int value) {
+
+            return forNumber(value);
+        }
+
+
+        /**
+         * @param value The numeric wire value of the corresponding enum entry.
+         * @return The enum associated with the given numeric wire value.
+         */
+        public static NodeType forNumber(int value) {
+
+            switch (value) {
+                case 0:
+                    return DOCUMENT;
+                case 1:
+                    return SECTION;
+                case 2:
+                    return SUPER_SECTION;
+                case 3:
+                    return HEADLINE;
+                case 4:
+                    return PARAGRAPH;
+                case 5:
+                    return TABLE;
+                case 6:
+                    return TABLE_CELL;
+                case 7:
+                    return IMAGE;
+                case 8:
+                    return HEADER;
+                case 9:
+                    return FOOTER;
+                default:
+                    return null;
+            }
+        }
+
+
+        public static com.google.protobuf.Internal.EnumLiteMap<NodeType> internalGetValueMap() {
+
+            return internalValueMap;
+        }
+
+
+        private static final com.google.protobuf.Internal.EnumLiteMap<NodeType> internalValueMap = new com.google.protobuf.Internal.EnumLiteMap<NodeType>() {
+            public NodeType findValueByNumber(int number) {
+
+                return NodeType.forNumber(number);
+            }
+        };
+
+
+        public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() {
+
+            if (this == UNRECOGNIZED) {
+                throw new IllegalStateException("Can't get the descriptor of an unrecognized enum value.");
+            }
+            return getDescriptor().getValues()
+                    .get(ordinal());
+        }
+
+
+        public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() {
+
+            return getDescriptor();
+        }
+
+
+        public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() {
+
+            return NodeTypeProto.getDescriptor().getEnumTypes()
+                    .get(0);
+        }
+
+
+        private static final NodeType[] VALUES = values();
+
+
+        public static NodeType valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+
+            if (desc.getType() != getDescriptor()) {
+                throw new IllegalArgumentException("EnumValueDescriptor is not for this type.");
+            }
+            if (desc.getIndex() == -1) {
+                return UNRECOGNIZED;
+            }
+            return VALUES[desc.getIndex()];
+        }
+
+
+        private final int value;
+
+
+        private NodeType(int value) {
+
+            this.value = value;
+        }
+
+        // @@protoc_insertion_point(enum_scope:NodeType)
+    }
+
+
+    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
+
+        return descriptor;
+    }
+
+
+    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
+
+    static {
+        String[] descriptorData = {"\n\016NodeType.proto*\223\001\n\010NodeType\022\014\n\010DOCUMEN"
+                                   + "T\020\000\022\013\n\007SECTION\020\001\022\021\n\rSUPER_SECTION\020\002\022\014\n\010H"
+                                   + "EADLINE\020\003\022\r\n\tPARAGRAPH\020\004\022\t\n\005TABLE\020\005\022\016\n\nT"
+                                   + "ABLE_CELL\020\006\022\t\n\005IMAGE\020\007\022\n\n\006HEADER\020\010\022\n\n\006FO"
+                                   + "OTER\020\tb\006proto3"};
+        descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[]{});
+        descriptor.resolveAllFeaturesImmutable();
+    }
+
+    // @@protoc_insertion_point(outer_class_scope)
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedSectionText.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Schema(description = "Object containing a simplified version, which contains almost exclusively text, of the document structure Section class.")
+public class SimplifiedSectionText {
+
+    @Schema(description = "The number of this Section. This is used to map the simplified section text back to the original Section.")
+    private String sectionNumber;
+    @Schema(description = "The text in this Section.")
+    private String text;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -0,0 +1,34 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Schema(description = "Object containing a simplified version, which contains almost exclusively text, of the document structure.")
+public class SimplifiedText {
+
+    @Schema(description = "Number of pages in the entire document.")
+    private int numberOfPages;
+    @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
+    @Builder.Default
+    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+    @Schema(description = "A list of the main section numbers ")
+    @Builder.Default
+    private List<String> mainSectionNumbers = new ArrayList<>();
+    @Schema(description = "A list of the header section numbers ")
+    @Builder.Default
+    private List<String> headerSectionNumbers = new ArrayList<>();
+    @Schema(description = "A list of the footer section numbers ")
+    @Builder.Default
+    private List<String> footerSectionNumbers = new ArrayList<>();
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/ParagraphData.java

@@ -2,20 +2,29 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
 
 import java.util.List;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 import lombok.Data;
 
 @Data
 @Builder
+@Schema(description = "Object containing information about a Paragraph/Headline/Header/Footer.")
 public class ParagraphData {
 
+    @Schema(description = "The text of this Semantic Node, without any linebreaks.", example = "This is some text.")
     private String text;
+    @Schema(description = "A list of text ranges in string offsets. Every character in any of the ranges is bold.", example = "[0, 15]")
     List<Range> boldTextBoundaries;
+    @Schema(description = "A list of text ranges in string offsets. Every character in any of the ranges is italic.", example = "[0, 15]")
     List<Range> italicTextBoundaries;
+    @Schema(description = "The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.", example = "[5, 10]")
     List<Integer> linebreaks;
+    @Schema(description = "The classification of this Paragraph.", allowableValues = "{paragraph, headline, header, footer}")
     private String classification;
 
+    @Schema(description = "Describes the text orientation of this semantic node. Any semantic node only has a single text orientation.", allowableValues = "{ZERO, QUARTER_CIRCLE, HALF_CIRCLE, THREE_QUARTER_CIRCLE}")
     private String orientation;
+    @Schema(description = "Describes the text direction in degrees of this semantic node. Any semantic node only has a single text direction.", minimum = "0", maximum = "359")
     private int textDirection;
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/Range.java

@@ -1,5 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
 
+import io.swagger.v3.oas.annotations.media.Schema;
+
+@Schema(description = "Object specifying the start and end offsets of a text range in string offsets.")
 public record Range(int start, int end) {
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/ResearchDocumentData.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
 
 import java.util.List;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -9,8 +10,12 @@ import lombok.Data;
 @Builder
 @Data
 @AllArgsConstructor
+@Schema(description = "Object containing a simplified version of the document structure. This simplified form only knows Paragraphs and Tables. The Paragraph Objects might be a Paragraph, Headline, Header or Footer.")
 public class ResearchDocumentData {
 
+    @Schema(description = "File name of the original uploaded file.")
     String originalFile;
+    @Schema(description = "A List of all paragraphs/headline or table objects, that have been parsed in this document.")
     List<StructureObject> structureObjects;
+
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/RowData.java

@@ -2,14 +2,19 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
 
 import java.util.List;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 
 @Data
 @AllArgsConstructor
+@Schema(description = "Object containing information about a Table Row.")
 public class RowData {
 
+    @Schema(description = "Boolean indicating whether this table row is classified as a header row.")
     boolean header;
+    @Schema(description = "A list of Objects containing information about the text in each cell of this row.")
     List<ParagraphData> cellText;
+    @Schema(description = "The bounding box of this StructureObject. Is always exactly 4 values representing x, y, w, h, where x, y specify the lower left corner.")
     float[] bBox;
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/StructureObject.java

@@ -1,5 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
 
+import java.util.List;
+
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -7,13 +10,22 @@ import lombok.Data;
 @Data
 @Builder
 @AllArgsConstructor
+@Schema(description = "Object containing information about either a Paragraph/Headline/Header/Footer or a Table.")
 public class StructureObject {
 
+    @Schema(description = "The ID of this StructureObject.")
     Integer structureObjectNumber;
+    @Schema(description = "The Tree ID of this StructureObject.")
+    List<Integer> treeId;
+    @Schema(description = "This value indicates the start of the string offsets in this Object, with respect to the reading order.")
     int page;
+    @Schema(description = "This stringOffset indicates the start of the string offsets in this Object, with respect to the reading order of the entire document. It is equal to the previous' StructureObject stringOffset + its length.")
     int stringOffset;
+    @Schema(description = "The bounding box of this StructureObject. Is always exactly 4 values representing x, y, w, h, where x, y specify the lower left corner.", example = "[100, 100, 50, 50]")
     float[] boundingBox;
+    @Schema(description = "Object containing information about a Paragraph/Headline/Header/Footer. Either this or table is null.")
     ParagraphData paragraph;
+    @Schema(description = "Object containing information about a Table. Either this or paragraph is null.")
     TableData table;
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/TableData.java

@@ -2,14 +2,20 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
 
 import java.util.List;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 
 @Data
 @AllArgsConstructor
+@Schema(description = "Object containing information about a Table.")
 public class TableData {
 
+    @Schema(description = "A list of Objects containing information about all rows in this table.")
     List<RowData> rowData;
+    @Schema(description = "Number of columns in this table.")
     Integer numberOfCols;
+    @Schema(description = "Number of rows in this table.")
     Integer numberOfRows;
+
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingFinishedEvent.java

@@ -2,9 +2,19 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 import java.util.Map;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 
 @Builder
-public record LayoutParsingFinishedEvent(Map<String, String> identifier, long duration, int numberOfPages, String message) {
+@Schema(description = "Object containing information about the layout parsing.")
+public record LayoutParsingFinishedEvent(
+        @Schema(description = "General purpose identifier. It is returned exactly the same way it is inserted with the LayoutParsingRequest.")
+        Map<String, String> identifier,//
+        @Schema(description = "The duration of a single layout parsing in ms.")
+        long duration,//
+        @Schema(description = "The number of pages of the parsed document.")
+        int numberOfPages,//
+        @Schema(description = "A general message. It contains some information useful for a developer, like the paths where the files are stored. Not meant to be machine readable.")
+        String message) {
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,7 +2,9 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
-    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_dead_letter_queue";
-    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE_PREFIX = "layout_parsing_request";
+    public static final String LAYOUT_PARSING_REQUEST_EXCHANGE = "layout_parsing_request_exchange";
+    public static final String LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX = "layout_parsing_response";
+    public static final String LAYOUT_PARSING_RESPONSE_EXCHANGE = "layout_parsing_response_exchange";
+    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_error";
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -3,21 +3,45 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 import java.util.Map;
 import java.util.Optional;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
-
+import lombok.NonNull;
 
 @Builder
+@Schema(description = "Object containing all storage paths the service needs to know.")
 public record LayoutParsingRequest(
-        LayoutParsingType layoutParsingType,
+        @Schema(description = "Enum specifying the type of layout parsing to be performed.", allowableValues = "{RedactManager, DocuMine, TAAS}")//
+        @NonNull LayoutParsingType layoutParsingType,
+
+        @Schema(description = "General purpose identifiers. They are not changed by the service at all and are returned as is in the response queue.")//
         Map<String, String> identifier,
-        String originFileStorageId,
-        Optional<String> tablesFileStorageId,
-        Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
-        String researchDocumentStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId,
-        String sectionGridStorageId) {
+
+        @Schema(description = "Path to the original PDF file.")//
+        @NonNull String originFileStorageId,//
+
+        @Schema(description = "Optional Path to the table extraction file.")//
+        Optional<String> tablesFileStorageId,//
+        @Schema(description = "Optional Path to the image classification file.")//
+        Optional<String> imagesFileStorageId,//
+
+        @Schema(description = "Optional Path to the the visual layout parsing service file") Optional<String> visualLayoutParsingFileId,//
+
+        @Schema(description = "Path where the Document Structure File will be stored.")//
+        @NonNull String structureFileStorageId,//
+        @Schema(description = "Path where the Research Data File will be stored.")//
+        String researchDocumentStorageId,//
+        @Schema(description = "Path where the Document Text File will be stored.")//
+        @NonNull String textBlockFileStorageId,//
+        @Schema(description = "Path where the Document Positions File will be stored.")//
+        @NonNull String positionBlockFileStorageId,//
+        @Schema(description = "Path where the Document Pages File will be stored.")//
+        @NonNull String pageFileStorageId,//
+        @Schema(description = "Path where the Document Markdown File will be stored.")//
+        Optional<String> documentMarkdownFileStorageId,//
+        @Schema(description = "Path where the Simplified Text File will be stored.")//
+        @NonNull String simplifiedTextStorageId,//
+        @Schema(description = "Path where the Viewer Document PDF will be stored.")//
+        @NonNull String viewerDocumentStorageId
+) {
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java

@@ -2,6 +2,11 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public enum LayoutParsingType {
     REDACT_MANAGER,
-    TAAS,
-    DOCUMINE
+    REDACT_MANAGER_OLD,
+    REDACT_MANAGER_PARAGRAPH_DEBUG,
+    REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+    DOCUMINE,
+    DOCUMINE_OLD,
+    CLARIFYND,
+    CLARIFYND_PARAGRAPH_DEBUG
 }

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentPage.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+message AllDocumentPages {
+
+  repeated DocumentPage documentPages = 1;
+}
+
+message DocumentPage {
+  // The page number, starting with 1.
+  int32 number = 1;
+
+  // The page height in PDF user units.
+  int32 height = 2;
+
+  // The page width in PDF user units.
+  int32 width = 3;
+
+  // The page rotation as specified by the PDF.
+  int32 rotation = 4;
+}
+

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentPositionData.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+message AllDocumentPositionData {
+
+  repeated DocumentPositionData documentPositionData = 1;
+}
+
+message DocumentPositionData {
+  // Identifier of the text block.
+  int64 id = 1;
+
+  // For each string coordinate in the search text of the text block, the array contains an entry relating the string coordinate to the position coordinate.
+  // This is required due to the text and position coordinates not being equal.
+  repeated int32 stringIdxToPositionIdx = 2;
+
+  // The bounding box for each glyph as a rectangle. This matrix is of size (n,4), where n is the number of glyphs in the text block.
+  // The second dimension specifies the rectangle with the value x, y, width, height, with x, y specifying the lower left corner.
+  // In order to access this information, the stringIdxToPositionIdx array must be used to transform the coordinates.
+  repeated Position positions = 3;
+
+  // Definition of a BoundingBox that contains x, y, width, and height.
+  message Position {
+    repeated float value = 1;
+  }
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentStructure.proto

@@ -0,0 +1,8 @@
+syntax = "proto3";
+
+import "EntryData.proto";
+
+message DocumentStructure {
+  // The root EntryData represents the Document.
+  EntryData root = 1;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/DocumentTextData.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+message AllDocumentTextData {
+
+  repeated DocumentTextData documentTextData = 1;
+}
+
+message DocumentTextData {
+  // Identifier of the text block.
+  int64 id = 1;
+
+  // The page the text block occurs on.
+  int64 page = 2;
+
+  // The text of the text block.
+  string searchText = 3;
+
+  // Each text block is assigned a number on a page, starting from 0.
+  int32 numberOnPage = 4;
+
+  // The text blocks are ordered, this number represents the start of the text block as a string offset.
+  int32 start = 5;
+
+  // The text blocks are ordered, this number represents the end of the text block as a string offset.
+  int32 end = 6;
+
+  // The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.
+  repeated int32 lineBreaks = 7;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/EntryData.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+import "LayoutEngine.proto";
+import "NodeType.proto";
+
+message EntryData {
+  // Type of the semantic node.
+  NodeType type = 1;
+
+  // Specifies the position in the parsed tree structure.
+  repeated int32 treeId = 2;
+
+  // Specifies the text block IDs associated with this semantic node.
+  repeated int64 atomicBlockIds = 3;
+
+  // Specifies the pages this semantic node appears on.
+  repeated int64 pageNumbers = 4;
+
+  // Some semantic nodes have additional information, this information is stored in this Map.
+  map<string, string> properties = 5;
+
+  // All child Entries of this Entry.
+  repeated EntryData children = 6;
+
+  // Describes the origin of the semantic node.
+  repeated LayoutEngine engines = 7;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/LayoutEngine.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+enum LayoutEngine {
+  ALGORITHM = 0;
+  AI = 1;
+  OUTLINE = 2;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/resources/NodeType.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+enum NodeType {
+  DOCUMENT = 0;
+  SECTION = 1;
+  SUPER_SECTION = 2;
+  HEADLINE = 3;
+  PARAGRAPH = 4;
+  TABLE = 5;
+  TABLE_CELL = 6;
+  IMAGE = 7;
+  HEADER = 8;
+  FOOTER = 9;
+}

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -0,0 +1,40 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.4"
+}
+
+description = "layoutparser-service-processor"
+
+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0"
+
+dependencies {
+    implementation(project(":layoutparser-service-internal-api"))
+    implementation(project(":viewer-doc-processor"))
+
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.564.0-RED9010.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
+    }
+    implementation("com.knecon.fforesight:tenant-commons:0.30.0") {
+        exclude("com.iqser.red.commons", "storage-commons")
+    }
+    implementation("com.iqser.red.commons:storage-commons:2.50.0")
+
+    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
+    implementation("org.jgrapht:jgrapht-core:1.5.2")
+    implementation("org.apache.pdfbox:jbig2-imageio:3.0.4")
+    implementation("com.github.jai-imageio:jai-imageio-core:1.4.0")
+    implementation("com.github.jai-imageio:jai-imageio-jpeg2000:1.4.0")
+    implementation("org.tinspin:tinspin-indexes:2.1.3")
+    implementation("org.commonmark:commonmark:0.22.0")
+    implementation("org.commonmark:commonmark-ext-gfm-tables:0.22.0")
+    implementation("com.pdftron:PDFNet:10.11.0")
+    implementation("org.apache.commons:commons-text:1.12.0")
+
+    implementation("com.google.protobuf:protobuf-java-util:4.27.1")
+}

layoutparser-service/layoutparser-service-processor/pom.xml

@@ -1,71 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>0.1-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>layoutparser-service-processor</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-shared-api-v1</artifactId>
-            <version>2.36.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>tenant-commons</artifactId>
-            <version>${tennat-commons.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>layoutparser-service-internal-api</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>storage-commons</artifactId>
-            <version>${storage-commons.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox-tools</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-afterburner</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-jsr310</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-web</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-amqp</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>RELEASE</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-</project>

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -2,140 +2,460 @@ package com.knecon.fforesight.service.layoutparser.processor;
 
 import static java.lang.String.format;
 
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Rectangle2D;
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
 
+import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.mapper.taas.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.MarkdownMapper;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
-import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineExtractorService;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineValidationService;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.TOCEnrichmentService;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableOfContents;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
+import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.TextRulingsClassifier;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.BlockificationPostprocessingService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.graphics.Box;
+import com.knecon.fforesight.service.layoutparser.processor.services.graphics.GraphicExtractorService;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
+@SuppressWarnings("PMD.CloseResource")
 @Slf4j
 @Service
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class LayoutParsingPipeline {
 
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final CvTableParsingAdapter cvTableParsingAdapter;
-    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final PdfParsingService pdfParsingService;
-    private final SectionsBuilderService sectionsBuilderService;
-    private final SectionGridCreatorService sectionGridCreatorService;
-    private final TaasClassificationService taasClassificationService;
-    private final RedactManagerClassificationService redactManagerClassificationService;
-    private final DocuMineClassificationService docuMineClassificationService;
+    ImageServiceResponseAdapter imageServiceResponseAdapter;
+    CvTableParsingAdapter cvTableParsingAdapter;
+    LayoutParsingStorageService layoutParsingStorageService;
+    SectionsBuilderService sectionsBuilderService;
+    SimplifiedSectionTextService simplifiedSectionTextService;
+    RulingCleaningService rulingCleaningService;
+    TableExtractionService tableExtractionService;
+    DocuMineBlockificationService docuMineBlockificationService;
+    RedactManagerBlockificationService redactManagerBlockificationService;
+    BlockificationPostprocessingService blockificationPostprocessingService;
+    DocstrumBlockificationService docstrumBlockificationService;
+    LayoutGridService layoutGridService;
+    ObservationRegistry observationRegistry;
+    VisualLayoutParsingAdapter visualLayoutParsingAdapter;
+    GraphicExtractorService graphicExtractorService;
+    OutlineExtractorService outlineExtractorService;
+    OutlineValidationService outlineValidationService;
+    TOCEnrichmentService tocEnrichmentService;
+    LayoutparserSettings settings;
+    ClassificationService classificationService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
 
         long start = System.currentTimeMillis();
-        PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
+        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());
 
-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
-        }
+        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
+        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);
 
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
-        }
+        VisualLayoutParsingResponse visualLayoutParsingResponse = layoutParsingRequest.visualLayoutParsingFileId()
+                .map(layoutParsingStorageService::getVisualLayoutParsingFile).orElse(new VisualLayoutParsingResponse());
+        ImageServiceResponse imageServiceResponse = layoutParsingRequest.imagesFileStorageId()
+                .map(layoutParsingStorageService::getImagesFile).orElse(new ImageServiceResponse());
+        TableServiceResponse tableServiceResponse = layoutParsingRequest.tablesFileStorageId()
+                .map(layoutParsingStorageService::getTablesFile).orElse(new TableServiceResponse());
 
-        Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
-        int numberOfPages = originDocument.getNumberOfPages();
-        originDocument.close();
+        ClassificationDocument classificationDocument = parseLayout(settings.getLayoutParsingTypeOverride() == null //
+                                                                            ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
+                                                                    originFile,
+                                                                    imageServiceResponse,
+                                                                    tableServiceResponse,
+                                                                    visualLayoutParsingResponse,
+                                                                    layoutParsingRequest.identifier());
+
+        log.info("Building document graph for {}", layoutParsingRequest.identifier());
+
+        Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null //
+                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument);
+
+        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
+
+        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false);
+
+        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
 
-        layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
         layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+        if (layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
+            layoutParsingStorageService.storeMarkdownFile(layoutParsingRequest.documentMarkdownFileStorageId().get(), new MarkdownMapper().toMarkdownContent(documentGraph));
+        }
+        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
 
-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+        if (layoutParsingRequest.researchDocumentStorageId() != null) {
+            log.info("Building research document data for {}", layoutParsingRequest.identifier());
             var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
             layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
         }
 
+        if (!viewerDocumentFile.equals(originFile)) {
+            assert !viewerDocumentFile.exists() || viewerDocumentFile.delete();
+        }
+        assert !originFile.exists() || originFile.delete();
+
         return LayoutParsingFinishedEvent.builder()
                 .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(numberOfPages)
+                .numberOfPages(documentGraph.getNumberOfPages())
                 .duration(System.currentTimeMillis() - start)
-                .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId()))
+                .message(format("""
+                                        Layout parsing has finished in %.02f s.
+                                        identifiers: %s
+                                        %s
+                                        Files have been saved with Ids:
+                                        Structure: %s
+                                        Text: %s
+                                        Positions: %s
+                                        PageData: %s
+                                        Simplified Text: %s
+                                        Viewer Doc: %s""",
+                                ((float) (System.currentTimeMillis() - start)) / 1000,
+                                layoutParsingRequest.identifier(),
+                                buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
+                                layoutParsingRequest.structureFileStorageId(),
+                                layoutParsingRequest.textBlockFileStorageId(),
+                                layoutParsingRequest.positionBlockFileStorageId(),
+                                layoutParsingRequest.pageFileStorageId(),
+                                layoutParsingRequest.simplifiedTextStorageId(),
+                                layoutParsingRequest.viewerDocumentStorageId()))
                 .build();
+
     }
 
 
-    public Document parseLayout(LayoutParsingType layoutParsingType,
-                                PDDocument originDocument,
-                                ImageServiceResponse imageServiceResponse,
-                                TableServiceResponse tableServiceResponse) {
+    private Document observeBuildDocumentGraph(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument) {
 
-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
-                originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
+        AtomicReference<Document> documentReference = new AtomicReference<>();
 
-        switch (layoutParsingType) {
-            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
-            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
-        }
+        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry)
+                .contextualName("build-document-graph")
+                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(layoutParsingType, classificationDocument)));
 
-        sectionsBuilderService.buildSections(classificationDocument);
-
-        return DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+        return documentReference.get();
     }
 
 
-    public Document parseLayoutWithTimer(LayoutParsingType layoutParsingType,
-                                         PDDocument originDocument,
-                                         ImageServiceResponse imageServiceResponse,
-                                         TableServiceResponse tableServiceResponse) {
+    private String buildSemanticNodeCountMessage(int numberOfPages, Map<NodeType, Long> semanticNodeCounts) {
 
-        long start = System.currentTimeMillis();
+        return format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
+                      numberOfPages,
+                      semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                      semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                      semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                      semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                      semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                      semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                      semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
+    }
 
-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType, originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
 
-        System.out.printf("parsed %d ms", System.currentTimeMillis() - start);
+    @SneakyThrows
+    @Observed(name = "LayoutParsingPipeline", contextualName = "parse-layout")
+    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
+                                              File originFile,
+                                              ImageServiceResponse imageServiceResponse,
+                                              TableServiceResponse tableServiceResponse,
+                                              VisualLayoutParsingResponse visualLayoutParsingResponse,
+                                              Map<String, String> identifier) {
 
-        start = System.currentTimeMillis();
-        switch (layoutParsingType) {
-            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
-            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
+        PDDocument originDocument = openDocument(originFile);
+        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
+
+        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
+        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
+        Map<Integer, List<ClassifiedImage>> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse);
+
+        ClassificationDocument classificationDocument = new ClassificationDocument();
+
+        if (settings.isDebug() || identifier.containsKey("debug")) {
+            classificationDocument.getLayoutDebugLayer().setActive(true);
         }
-        System.out.printf(", classified %d ms", System.currentTimeMillis() - start);
 
-        start = System.currentTimeMillis();
-        sectionsBuilderService.buildSections(classificationDocument);
-        System.out.printf(", sections built %d ms", System.currentTimeMillis() - start);
+        List<ClassificationPage> classificationPages = new ArrayList<>();
 
-        start = System.currentTimeMillis();
-        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-        System.out.printf(", graph constructed %d ms", System.currentTimeMillis() - start);
+        classificationDocument.setOutlineObjectTree(outlineExtractorService.getOutlineObjectTree(originDocument));
+
+        long pageCount = originDocument.getNumberOfPages();
+
+        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+
+            if (pageNumber % 100 == 0) {
+                // re-open document every once in a while to save on RAM. This has no significant performance impact.
+                // This is due to PDFBox caching all images and some other stuff with Soft References. This dereferences them and forces the freeing of memory.
+                originDocument.close();
+                originDocument = openDocument(originFile);
+            }
+
+            if (pageNumber % 100 == 0 || pageNumber == pageCount || pageNumber == 1) {
+                log.info("Extracting text on Page {} for {}", pageNumber, identifier);
+            }
+
+            classificationDocument.setPages(classificationPages);
+            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+            PDPage pdPage = originDocument.getPage(pageNumber - 1);
+            stripper.setPageNumber(pageNumber);
+            stripper.setStartPage(pageNumber);
+            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
+            stripper.getText(originDocument);
+            List<Word> words = stripper.getWords();
+
+//            rotateDirAdjExactly(words, pdPage); // works really well for many highly rotated documents (e.g. VV-331340.pdf), but it decreases the headline performance by 1.3%, so I am leaving it out for now
+
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
+                var lines = TextPositionOperations.groupByLine(new HashSet<>(words));
+                classificationDocument.getLayoutDebugLayer().addLineVisualizationsFromNestedTextPosition(lines, pageNumber);
+                words = TextPositionOperations.sortWords(lines);
+            }
+            classificationDocument.getLayoutDebugLayer().addTextVisualizations(words, pageNumber);
+
+            PDRectangle pdr = pdPage.getMediaBox();
+
+            List<Ruling> rulings = stripper.getRulings();
+            classificationDocument.getLayoutDebugLayer().addRulingVisualization(rulings, pageNumber);
+            CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(pdfTableCells.get(pageNumber), rulings);
+
+            PageInformation pageInformation = PageInformation.fromPDPage(pageNumber, pdPage);
+            List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontals(), cleanRulings.getVerticals(), pageInformation);
+            classificationDocument.getLayoutDebugLayer().addCellVisualizations(emptyTableCells, pageNumber);
+
+            TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(words, cleanRulings);
+
+            List<Box> graphics = graphicExtractorService.extractPathElementGraphics(originDocument, pdPage, pageNumber, cleanRulings, stripper.getWords(), false);
+
+            pdfImages.computeIfAbsent(pageNumber, x -> new ArrayList<>())
+                    .addAll(graphics.stream()
+                                    .map(g -> new ClassifiedImage(new Rectangle2D.Double(g.x1, g.y1, g.width(), g.height()),
+                                                                  ImageType.GRAPHIC,
+                                                                  false,
+                                                                  stripper.getPageNumber(),
+                                                                  ""))
+                                    .toList());
+
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER_OLD -> redactManagerBlockificationService.blockify(stripper.getWords(), cleanRulings, classificationDocument.getLayoutDebugLayer());
+                case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
+                case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
+                        docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
+                case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG ->
+                        docstrumBlockificationService.blockify(words, cleanRulings, false, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
+            };
+
+            updateClassificationPage(pdPage, pdr, classificationPage, cleanRulings, pageNumber, pageInformation);
+
+            blockificationPostprocessingService.findHeadlinesFromOutline(classificationDocument, pageNumber, classificationPage, pageInformation);
+
+            classificationDocument.getLayoutDebugLayer().addMarkedContentVisualizations(stripper.getMarkedContents(), pageNumber);
+            // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
+            classificationPage.setMarkedContentBboxPerType(convertMarkedContents(stripper.getMarkedContents()));
+
+            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+            if (pdfImages.containsKey(pageNumber)) {
+                classificationPage.setImages(pdfImages.get(pageNumber));
+                imageServiceResponseAdapter.findOcr(classificationPage);
+            }
+
+            if (signatures.containsKey(pageNumber)) {
+                if (classificationPage.getImages() == null || classificationPage.getImages().isEmpty()) {
+                    classificationPage.setImages(signatures.get(pageNumber));
+                } else {
+                    classificationPage.getImages().addAll(signatures.get(pageNumber));
+                }
+            }
+
+            tableExtractionService.extractTables(emptyTableCells, classificationPage);
+
+            buildPageStatistics(classificationPage);
+            increaseDocumentStatistics(classificationPage, classificationDocument);
+
+            classificationPages.add(classificationPage);
+        }
+
+        originDocument.close();
+
+        classificationService.classify(classificationDocument, layoutParsingType, identifier);
+
+        TableOfContents tableOfContents = outlineValidationService.createToC(classificationDocument);
+        classificationDocument.setTableOfContents(tableOfContents);
+
+        log.info("Building Sections for {}", identifier);
+
+        switch (layoutParsingType) {
+            case CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_PARAGRAPH_DEBUG -> sectionsBuilderService.buildParagraphDebugSections(classificationDocument);
+            default -> tocEnrichmentService.assignSectionBlocksAndImages(classificationDocument);
+        }
+
+        return classificationDocument;
+    }
+
+
+    private static void updateClassificationPage(PDPage pdPage,
+                                                 PDRectangle pdr,
+                                                 ClassificationPage classificationPage,
+                                                 CleanRulings cleanRulings,
+                                                 int pageNumber,
+                                                 PageInformation pageInformation) {
+
+        int rotation = pdPage.getRotation();
+        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+        classificationPage.setCleanRulings(cleanRulings);
+        classificationPage.setRotation(rotation);
+        classificationPage.setLandscape(isLandscape);
+        classificationPage.setPageNumber(pageNumber);
+        classificationPage.setPageWidth((float) pageInformation.width());
+        classificationPage.setPageHeight((float) pageInformation.height());
+    }
+
+
+    private static void rotateDirAdjExactly(List<Word> words, PDPage pdPage) {
+
+        for (TextDirection dir : TextDirection.values()) {
+            double averageRotation = words.stream()
+                    .map(Word::getCharacters)
+                    .flatMap(Collection::stream)
+                    .map(Character::getTextPosition)
+                    .filter(pos -> pos.getDir().equals(dir))
+                    .mapToDouble(RedTextPosition::getExactDir).average().orElse(0);
+
+            if (averageRotation == 0) {
+                continue;
+            }
+
+            AffineTransform rotateInstance = AffineTransform.getRotateInstance(averageRotation, pdPage.getMediaBox().getWidth() / 2, pdPage.getMediaBox().getHeight() / 2);
+
+            for (Word word : words) {
+                if (!dir.equals(word.getDir())) {
+                    continue;
+                }
+                word.transform(rotateInstance);
+            }
+        }
+    }
+
+
+    private void addNumberOfPagesToTrace(int numberOfPages, long size) {
+
+        if (observationRegistry.getCurrentObservation() != null) {
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileSize", String.valueOf(size));
+        }
+    }
+
+
+    @SneakyThrows
+    private PDDocument openDocument(File originFile) {
+
+        PDDocument document = Loader.loadPDF(originFile);
+        document.setAllSecurityToBeRemoved(true);
         return document;
     }
 
+
+    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+
+        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
+        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
+        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
+        return markedContentBboxes;
+    }
+
+
+    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
+
+        document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
+        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
+        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
+    }
+
+
+    private void buildPageStatistics(ClassificationPage classificationPage) {
+
+        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
+        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                if (((TextPageBlock) textBlock).getWords() == null) {
+                    continue;
+                }
+                for (Word word : ((TextPageBlock) textBlock).getWords()) {
+                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
+                    classificationPage.getFontCounter().add(word.getFont());
+                    classificationPage.getFontSizeCounter().add(word.getFontSize());
+                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
+                }
+            }
+        }
+
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingServiceProcessorConfiguration.java

@@ -1,10 +1,23 @@
 package com.knecon.fforesight.service.layoutparser.processor;
 
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 
+import com.knecon.fforesight.service.viewerdoc.service.PDFTronViewerDocumentService;
+
+import io.micrometer.observation.ObservationRegistry;
+
 @Configuration
 @ComponentScan
 public class LayoutParsingServiceProcessorConfiguration {
 
+    @Bean
+    @Autowired
+    public PDFTronViewerDocumentService viewerDocumentService(ObservationRegistry registry) {
+
+        return new PDFTronViewerDocumentService(registry);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -1,32 +1,36 @@
 package com.knecon.fforesight.service.layoutparser.processor;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
-import java.io.FileOutputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
 
-import org.apache.commons.io.IOUtils;
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.io.MemoryUsageSetting;
-import org.apache.pdfbox.pdmodel.PDDocument;
+import org.springframework.core.task.TaskExecutor;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocVersioningUtility;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -37,50 +41,98 @@ public class LayoutParsingStorageService {
     private final StorageService storageService;
     private final ObjectMapper objectMapper;
 
+    private final TaskExecutor taskExecutor;
 
-    public PDDocument getOriginFile(String storageId) throws IOException {
 
-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-            File tempFile = createTempFile("document", ".pdf");
-            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
-                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
-            }
-            return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
+    @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
+    public File getOriginFile(String storageId) throws IOException {
+
+        File tempFile = createTempFile("document", ".pdf");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return tempFile;
+    }
+
+
+    @Observed(name = "LayoutParsingStorageService", contextualName = "get-viewer-doc-file")
+    public Optional<File> getViewerDocFile(String storageId) throws IOException {
+
+        if (!storageService.objectExists(TenantContext.getTenantId(), storageId)) {
+            return Optional.empty();
+        }
+        File tempFile = createTempFile("viewerDocument", ".pdf");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+
+        if (!ViewerDocVersioningUtility.isCurrentVersion(tempFile)) {
+            assert tempFile.delete();
+            return Optional.empty();
+        }
+
+        return Optional.of(tempFile);
+    }
+
+
+    @SneakyThrows
+    public ImageServiceResponse getImagesFile(String storageId) {
+
+        try (InputStream inputStream = getObject(storageId)) {
+
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
         }
     }
 
 
-    public ImageServiceResponse getImagesFile(String storageId) throws IOException {
+    @SneakyThrows
+    public TableServiceResponse getTablesFile(String storageId) {
 
-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var tableClassificationStream = getObject(storageId)) {
 
-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
         }
     }
 
 
-    public TableServiceResponse getTablesFile(String storageId) throws IOException {
-
-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+@SneakyThrows
+    public VisualLayoutParsingResponse getVisualLayoutParsingFile(String storageId) {
 
+        try (InputStream inputStream = getObject(storageId)) {
+            return objectMapper.readValue(inputStream, VisualLayoutParsingResponse.class);
         }
     }
 
 
+    @SneakyThrows
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
     public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {
 
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getDocumentTextData());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getDocumentPositions());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getDocumentPages());
-    }
+        Runnable storeDocumentStructureRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
+                                                                                       layoutParsingRequest.structureFileStorageId(),
+                                                                                       documentData.getDocumentStructure());
 
+        CompletableFuture<Void> storeDocumentStructureFuture = CompletableFuture.runAsync(storeDocumentStructureRunnable, taskExecutor);
 
-    public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) {
+        Runnable storeDocumentTextDataRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
+                                                                                      layoutParsingRequest.textBlockFileStorageId(),
+                                                                                      documentData.getDocumentTextData());
 
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid);
+        CompletableFuture<Void> storeDocumentTextDataFuture = CompletableFuture.runAsync(storeDocumentTextDataRunnable, taskExecutor);
+
+        Runnable storeDocumentPositionsRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
+                                                                                       layoutParsingRequest.positionBlockFileStorageId(),
+                                                                                       documentData.getDocumentPositions());
+
+        CompletableFuture<Void> storeDocumentPositionsFuture = CompletableFuture.runAsync(storeDocumentPositionsRunnable, taskExecutor);
+
+        Runnable storeDocumentPagesRunnable = () -> storageService.storeProtoObject(TenantContext.getTenantId(),
+                                                                                   layoutParsingRequest.pageFileStorageId(),
+                                                                                   documentData.getDocumentPages());
+
+        CompletableFuture<Void> storeDocumentPagesFuture = CompletableFuture.runAsync(storeDocumentPagesRunnable, taskExecutor);
+
+        CompletableFuture.allOf(storeDocumentStructureFuture, storeDocumentTextDataFuture, storeDocumentPositionsFuture, storeDocumentPagesFuture).join();
     }
 
 
@@ -90,26 +142,6 @@ public class LayoutParsingStorageService {
     }
 
 
-    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
-
-        DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class);
-        DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.textBlockFileStorageId(),
-                DocumentTextData[].class);
-        DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.positionBlockFileStorageId(),
-                DocumentPositionData[].class);
-        DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class);
-
-        return DocumentData.builder()
-                .documentStructure(tableOfContentsData)
-                .documentPositions(atomicPositionBlockData)
-                .documentTextData(documentTextDataBlockData)
-                .documentPages(documentPageData)
-                .build();
-    }
-
-
     private File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {
 
         File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();
@@ -134,4 +166,43 @@ public class LayoutParsingStorageService {
         }
     }
 
+
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-simplified-text")
+    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
+    }
+
+
+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
+
+    @SneakyThrows
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-viewer-document")
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, File out) {
+
+        try (var in = new FileInputStream(out)) {
+            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
+        }
+    }
+
+
+    @SneakyThrows
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-markdown-file")
+    public void storeMarkdownFile(String markdownFileStorageId, String markdownContent) {
+
+        try (InputStream inputStream = new ByteArrayInputStream(markdownContent.getBytes(StandardCharsets.UTF_8))) {
+
+            storageService.storeObject(TenantContext.getTenantId(), markdownFileStorageId, inputStream);
+        }
+
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutparserSettings.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Configuration
+@ConfigurationProperties("layoutparser")
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class LayoutparserSettings {
+
+    boolean debug;
+    LayoutParsingType layoutParsingTypeOverride;
+    String pdftronLicense;
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/CvTableParsingAdapter.java

@@ -1,49 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class CvTableParsingAdapter {
-
-    public Map<Integer, List<TableCells>> buildCvParsedTablesPerPage(TableServiceResponse tableServiceResponse) {
-
-        Map<Integer, List<TableCells>> tableCells = new HashMap<>();
-        tableServiceResponse.getData()
-                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells())));
-
-        return tableCells;
-    }
-
-
-    private Collection<TableCells> convertTableCells(List<TableCells> tableCells) {
-
-        List<TableCells> cvParsedTableCells = new ArrayList<>();
-
-        tableCells.forEach(t -> cvParsedTableCells.add(TableCells.builder()
-                .y0(t.getY0())
-                .x1(t.getX1())
-                .y1(t.getY1())
-                .x0(t.getX0())
-                .width(t.getWidth())
-                .height(t.getHeight())
-                .build()));
-
-        return cvParsedTableCells;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Classification.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import lombok.Data;
-
-@Data
-public class Classification {
-
-    private Map<String, Float> probabilities = new HashMap<>();
-    private String label;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/FilterGeometry.java

@@ -1,11 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class FilterGeometry {
-
-    private ImageSize imageSize;
-    private ImageFormat imageFormat;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Filters.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Filters {
-
-    private FilterGeometry geometry;
-    private Probability probability;
-    private boolean allPassed;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Geometry.java

@@ -1,11 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Geometry {
-
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageFormat.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageFormat {
-
-    private float quotient;
-    private boolean tooTall;
-    private boolean tooWide;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageMetadata.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageMetadata {
-
-    private Classification classification;
-    private Position position;
-    private Geometry geometry;
-    private Filters filters;
-    private boolean alpha;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageSize.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageSize {
-
-    private float quotient;
-    private boolean tooLarge;
-    private boolean tooSmall;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Position.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Position {
-
-    private float x1;
-    private float x2;
-    private float y1;
-    private float y2;
-    private int pageNumber;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Probability.java

@@ -1,10 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Probability {
-
-    private boolean unconfident;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/PageInfo.java

@@ -1,13 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import lombok.Data;
-
-@Data
-public class PageInfo {
-
-    private int number;
-    private int rotation;
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/TableData.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import lombok.Data;
-
-@Data
-public class TableData {
-
-    private PageInfo pageInfo;
-    private List<TableCells> tableCells = new ArrayList<>();
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/DocstrumSegmentationService.java

@@ -0,0 +1,98 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum;
+
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.LineBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.NearestNeighbourService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.ReadingOrderService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.SpacingService;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.service.ZoneBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutDebugLayer;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class DocstrumSegmentationService {
+
+    public static final double SAME_DIRECTION_THRESHOLD = 0.9;
+    private final NearestNeighbourService nearestNeighbourService;
+    private final SpacingService spacingService;
+    private final LineBuilderService lineBuilderService;
+    private final ZoneBuilderService zoneBuilderService;
+    private final ReadingOrderService readingOrderService;
+
+
+    public List<Zone> segmentPage(List<Word> textPositions, boolean xyOrder, CleanRulings usedRulings, LayoutDebugLayer visualizations) {
+
+        EnumMap<TextDirection, Integer> directionCounts = new EnumMap<>(TextDirection.class);
+
+        List<Zone> newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.ZERO);
+        directionCounts.put(TextDirection.ZERO, newZones.size());
+        List<Zone> zones = new ArrayList<>(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.QUARTER_CIRCLE);
+        directionCounts.put(TextDirection.QUARTER_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.HALF_CIRCLE);
+        directionCounts.put(TextDirection.HALF_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        newZones = computeZones(textPositions, usedRulings, visualizations, TextDirection.THREE_QUARTER_CIRCLE);
+        directionCounts.put(TextDirection.THREE_QUARTER_CIRCLE, newZones.size());
+        zones.addAll(newZones);
+
+        return readingOrderService.resolve(zones, xyOrder, mostSameDirection(directionCounts));
+    }
+
+
+    private boolean mostSameDirection(EnumMap<TextDirection, Integer> directionCounts) {
+
+        int total = directionCounts.values()
+                .stream()
+                .mapToInt(i -> i).sum();
+
+        if ((double) directionCounts.get(TextDirection.ZERO) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.QUARTER_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.HALF_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        } else if ((double) directionCounts.get(TextDirection.THREE_QUARTER_CIRCLE) / total > SAME_DIRECTION_THRESHOLD) {
+            return true;
+        }
+        return false;
+    }
+
+
+    private List<Zone> computeZones(List<Word> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
+
+        List<Character> characters = textPositions.stream()
+                .filter(t -> t.getDir() == direction)
+                .map(Word::getCharacters)
+                .flatMap(List::stream)
+                .toList();
+
+        nearestNeighbourService.findNearestNeighbors(characters);
+
+        double characterSpacing = spacingService.computeCharacterSpacing(characters);
+        double lineSpacing = Math.min(spacingService.computeLineSpacing(characters), 20);
+
+        List<Line> lines = lineBuilderService.buildLines(characters, characterSpacing, lineSpacing, rulings);
+        return zoneBuilderService.buildZones(lines, characterSpacing, lineSpacing, rulings);
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/AngleFilter.java

@@ -0,0 +1,31 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+public class AngleFilter {
+
+    protected double lowerAngle;
+    protected double upperAngle;
+
+
+    public AngleFilter(double lowerAngle, double upperAngle) {
+
+        this.lowerAngle = lowerAngle < -Math.PI / 2 ? lowerAngle + Math.PI : lowerAngle;
+        this.upperAngle = upperAngle >= Math.PI / 2 ? upperAngle - Math.PI : upperAngle;
+    }
+
+
+    public boolean matches(Neighbor neighbor) {
+
+        return matches(neighbor.getAngle());
+    }
+
+
+    public boolean matches(double angle) {
+
+        if (lowerAngle <= upperAngle) {
+            return lowerAngle <= angle && angle < upperAngle;
+        } else {
+            return lowerAngle <= angle || angle < upperAngle;
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/BoundingBox.java

@@ -0,0 +1,279 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Comparator;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@NoArgsConstructor
+public abstract class BoundingBox {
+
+    // Java coordinate system: (0, 0) is always upper left, x is increasing left to right and y is increasing from top to bottom.
+    // should be used when determining reading order or other tasks which require coordinates in a harmonized system.
+    protected Rectangle2D bBox; // I would not trust this coordinate when comparing rulings and text, due to the text positions being slightly off.
+
+    // PDF coordinate system: depends on page rotation, (0, 0) is lower left corner, x is increasing left to right and y from bottom to top.
+    // This rotates completely in 90 degree steps with page rotation.
+    // Needs to be used when writing to a PDF.
+    // Also, these are definitely correct and should be used whenever possible.
+    protected Rectangle2D bBoxPdf;
+
+    protected static final float VERTICAL_COMPARISON_THRESHOLD = 0.4f;
+
+
+    public double getX() {
+
+        return bBox.getX();
+    }
+
+
+    public double getY() {
+
+        return bBox.getY();
+    }
+
+
+    public double getMinX() {
+
+        return bBox.getMinX();
+    }
+
+
+    public double getMinY() {
+
+        return bBox.getMinY();
+    }
+
+
+    public double getPdfMinX() {
+
+        return bBoxPdf.getMinX();
+    }
+
+
+    public double getPdfMaxX() {
+
+        return bBoxPdf.getMaxX();
+    }
+
+
+    public double getPdfMinY() {
+
+        return bBoxPdf.getMinY();
+    }
+
+
+    public double getPdfMaxY() {
+
+        return bBoxPdf.getMaxY();
+    }
+
+
+    public double getWidth() {
+
+        return bBox.getWidth();
+    }
+
+
+    public double getHeight() {
+
+        return bBox.getHeight();
+    }
+
+
+    public double getMaxX() {
+
+        return bBox.getMaxX();
+    }
+
+
+    public double getMaxY() {
+
+        return bBox.getMaxY();
+    }
+
+
+    public double getArea() {
+
+        return (bBox.getHeight() * bBox.getWidth());
+    }
+
+
+    public boolean contains(BoundingBox contained) {
+
+        return contains(contained, 0);
+    }
+
+
+    public boolean contains(BoundingBox contained, double tolerance) {
+
+        return getPdfMinX() <= contained.getPdfMinX() + tolerance
+               && getPdfMinY() <= contained.getPdfMinY() + tolerance
+               && getPdfMaxX() >= contained.getPdfMaxX() - tolerance
+               && getPdfMaxY() >= contained.getPdfMaxY() - tolerance;
+    }
+
+
+    public boolean intersects(BoundingBox other) {
+
+        return this.intersectsX(other) && this.intersectsY(other);
+    }
+
+
+    public boolean intersects(BoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsX(other, xThreshold) && this.intersectsY(other, yThreshold);
+    }
+
+
+    public boolean intersectsX(BoundingBox other, float threshold) {
+
+        return this.getX() - threshold <= other.getMaxX() && this.getMaxX() + threshold >= other.getX();
+    }
+
+
+    public boolean intersectsPdf(BoundingBox other) {
+
+        return this.intersectsXPdf(other) && this.intersectsYPdf(other);
+    }
+
+
+    public boolean intersectsPdf(BoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsXPdf(other, xThreshold) && this.intersectsYPdf(other, yThreshold);
+    }
+
+
+    public boolean intersectsYPdf(BoundingBox other) {
+
+        return this.getPdfMinY() <= other.getPdfMaxY() && this.getPdfMaxY() >= other.getPdfMinY();
+    }
+
+
+    public boolean intersectsY(BoundingBox other) {
+
+        return this.getY() <= other.getMaxY() && this.getMaxY() >= other.getY();
+    }
+
+
+    public boolean intersectsY(BoundingBox other, float threshold) {
+
+        return this.getY() - threshold <= other.getMaxY() && this.getMaxY() + threshold >= other.getY();
+    }
+
+
+    public boolean intersectsYPdf(BoundingBox other, float threshold) {
+
+        return this.getPdfMinY() - threshold <= other.getPdfMaxY() && this.getPdfMaxY() + threshold >= other.getPdfMinY();
+    }
+
+
+    public boolean intersectsXPdf(BoundingBox other) {
+
+        return this.getPdfMinX() <= other.getPdfMaxX() && this.getPdfMaxX() >= other.getPdfMinX();
+    }
+
+
+    public boolean intersectsX(BoundingBox other) {
+
+        return this.getX() <= other.getMaxX() && this.getMaxX() >= other.getMinX();
+    }
+
+
+    public boolean intersectsXPdf(BoundingBox other, float threshold) {
+
+        return this.getPdfMinX() - threshold <= other.getPdfMaxX() && this.getMaxX() + threshold >= other.getPdfMinX();
+    }
+
+
+    public void setToBBoxOfComponents(List<? extends BoundingBox> components) {
+
+        this.bBox = components.stream()
+                .map(BoundingBox::getBBox)
+                .collect(RectangleTransformations.collectBBox());
+        this.bBoxPdf = components.stream()
+                .map(BoundingBox::getBBoxPdf)
+                .collect(RectangleTransformations.collectBBox());
+    }
+
+
+    public double verticalOverlap(BoundingBox other) {
+
+        return Math.max(0, Math.min(this.getPdfMaxY(), other.getPdfMaxY()) - Math.max(this.getPdfMinY(), other.getPdfMinY()));
+    }
+
+
+    public static final Comparator<BoundingBox> ILL_DEFINED_ORDER = (o1, o2) -> {
+
+        if (o1.equals(o2)) {
+            return 0;
+        }
+        if (o1.verticalOverlap(o2) > VERTICAL_COMPARISON_THRESHOLD * ((o1.getHeight() + o2.getHeight()) / 2)) {
+            return Double.compare(o1.getPdfMinX(), o2.getPdfMinX());
+        } else {
+            return Double.compare(o1.getPdfMaxY(), o2.getPdfMaxY());
+        }
+    };
+
+
+    public double horizontalDistance(BoundingBox other) {
+
+        double rect1Right = getMaxX();
+        double rect1Left = getMinX();
+        double rect2Right = other.getMaxX();
+        double rect2Left = other.getMinX();
+
+        if (rect1Left > rect2Right || rect2Left > rect1Right) {
+            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public double verticalDistance(BoundingBox other) {
+
+        double rect1Top = getMaxY();
+        double rect1Bottom = getMinY();
+        double rect2Top = other.getMaxY();
+        double rect2Bottom = other.getMinY();
+
+        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
+            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public boolean rightOf(BoundingBox other) {
+
+        return this.intersectsY(other) && other.getMaxX() <= this.getMinX();
+    }
+
+
+    public boolean leftOf(BoundingBox other) {
+
+        return this.intersectsY(other) && other.getMinX() >= this.getMaxX();
+    }
+
+
+    public boolean isAbove(BoundingBox other) {
+
+        return this.intersectsX(other) && other.getMinY() >= this.getMaxY();
+    }
+
+
+    public boolean isBelow(BoundingBox other) {
+
+        return this.intersectsX(other) && this.getMinY() >= other.getMaxY();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Character.java

@@ -0,0 +1,86 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.Setter;
+
+@Data
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
+public class Character {
+
+    @EqualsAndHashCode.Include
+    private final double x;
+    @EqualsAndHashCode.Include
+    private final double y;
+    private final RedTextPosition textPosition;
+
+    @Setter
+    private List<Neighbor> neighbors = new ArrayList<>();
+
+
+    public Character(RedTextPosition chunk) {
+
+        this.x = chunk.getBBoxDirAdj().getCenterX();
+        this.y = chunk.getBBoxDirAdj().getCenterY();
+        this.textPosition = chunk;
+    }
+
+
+    public double getHeight() {
+
+        return textPosition.getHeightDirAdj();
+    }
+
+
+    public double distance(Character character) {
+
+        double dx = getX() - character.getX();
+        double dy = getY() - character.getY();
+        return Math.sqrt(dx * dx + dy * dy);
+    }
+
+
+    public double horizontalDistance(Character character) {
+
+        return Math.abs(getX() - character.getX());
+    }
+
+
+    public double verticalDistance(Character character) {
+
+        return Math.abs(getY() - character.getY());
+    }
+
+
+    public double overlappingDistance(Character other) {
+
+        double[] xs = new double[4];
+        double s = Math.sin(-0);
+        double c = Math.cos(-0);
+        xs[0] = c * x - s * y;
+        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDirAdj());
+        xs[2] = c * other.x - s * other.y;
+        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDirAdj());
+        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
+        Arrays.sort(xs);
+        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
+    }
+
+
+    public double angle(Character character) {
+
+        if (getX() > character.getX()) {
+            return FastAtan2.fastAtan2(getY() - character.getY(), getX() - character.getX());
+        } else {
+            return FastAtan2.fastAtan2(character.getY() - getY(), character.getX() - getX());
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/ColumnDetector.java

@@ -0,0 +1,324 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+
+/*
+WIP, mostly working, needs to be tested a bit more
+ */
+public class ColumnDetector {
+
+    public static final double MAX_VALUE_THRESHOLD = 0.5;
+    final static int bins_num = 512;
+    final static int globalStartIdx = 0; // ignore outer parts completely, we don't expect columns there
+    final static int globalEndIdx = bins_num; // i chose 7, since thirds seems a likely split for columns, therefore divided by 6 would eliminate those.
+    public static final double DERIVATIVE_ZERO_THRESHOLD = 1e-10;
+    public static final double MINIMUM_THRESHOLD_FOR_COLUMNS = 0.05;
+    public static final double NEAR_GLOBAL_THRESHOLD = 0.5;
+    double minY;
+    double maxY;
+    double midY;
+    double[] histogram;
+    double min;
+    double max;
+    double resolution;
+    double sum;
+    int N;
+
+
+    public ColumnDetector(double min, double max, double minY, double maxY) {
+
+        this.min = min;
+        this.max = max;
+        this.minY = minY;
+        this.maxY = maxY;
+        this.midY = maxY - minY;
+        this.resolution = (max - min) / bins_num;
+        this.histogram = new double[bins_num];
+    }
+
+
+    public void add(BoundingBox zone) {
+
+        N++;
+        double weight = computeWeight(zone);
+        int start = (int) ((zone.getMinX() - min) / resolution);
+        int end = (int) ((zone.getMaxX() - min) / resolution);
+        for (int i = start; i < end; i++) {
+            histogram[i] += weight;
+            sum += histogram[i];
+        }
+    }
+
+
+    private double computeWeight(BoundingBox zone) {
+
+        double areaWeight = zone.getBBox().getHeight();
+
+        double relativeDistance = relativeDistanceToMiddle(zone.getBBox().getCenterY());
+
+        double distanceWeight;
+        if (relativeDistance < 0.6) {
+            distanceWeight = 1;
+        } else if (relativeDistance < 0.8) {
+            distanceWeight = 0.8;
+        } else {
+            distanceWeight = 0.1;
+        }
+
+        return areaWeight * distanceWeight;
+    }
+
+
+    private double relativeDistanceToMiddle(double y) {
+
+        double range = (maxY - minY) / 2;
+        double mid = minY + range;
+
+        return Math.abs(y - mid) / range;
+    }
+
+
+    public double[] computeDerivative() {
+
+        int length = histogram.length;
+        double[] derivative = new double[length];
+
+        for (int i = 0; i < length; i++) {
+            if (i == 0) {
+                derivative[i] = (histogram[i + 1] - histogram[i]) / resolution;
+            } else if (i == length - 1) {
+                derivative[i] = (histogram[i] - histogram[i - 1]) / resolution;
+            } else {
+                derivative[i] = (histogram[i + 1] - histogram[i - 1]) / (2 * resolution);
+            }
+        }
+
+        return derivative;
+    }
+
+
+    public double calcMean(double[] arr, int start, int end) {
+
+        if (start == end) {
+            return 0;
+        }
+        double sum = 0;
+        for (int i = start; i < end; i++) {
+            sum += arr[i];
+        }
+        return sum / (end - start);
+    }
+
+
+    /*
+    Find columns, by finding all local maxima/minima of the derivative. Filtering them for the ones with the biggest values.
+    For each found minima, we will step to the right until we hit a 0 in the derivative, this indicates a minimum in the main histogram. If this minimum is below a threshold, it is deemed a column divider.
+    Same goes for maxima, but stepping to the left now, since minima in the function will always be to the left of a maximum in its derivative.
+     */
+    public List<Double> determineColumnsWithDerivative(double[] derivative) {
+
+        assert derivative.length == histogram.length;
+
+        Set<Integer> columnIndices = new HashSet<>();
+        double mean = calcMean(histogram, 0, histogram.length);
+        double maxDvValue = calcMax(derivative);
+        double minDvValue = calcMin(derivative);
+
+        if (maxDvValue - minDvValue < mean * MAX_VALUE_THRESHOLD) {
+            Collections.emptyList();
+        }
+
+        Extrema derivativeExtrema = calculateNearGlobalExtrema(derivative, maxDvValue, minDvValue);
+
+        List<Integer> columnsRightOfMinima = findZerosToTheRightOfMinima(derivative, derivativeExtrema.minima(), mean);
+        columnIndices.addAll(columnsRightOfMinima);
+
+        List<Integer> columnsLeftOfMaxima = findZerosToTheLeftOfMaxima(derivative, derivativeExtrema.maxima(), mean);
+        columnIndices.addAll(columnsLeftOfMaxima);
+
+        return columnIndices.stream()
+                .sorted(Comparator.naturalOrder())
+                .map(this::calculateXCoordinateFromIdx)
+                .toList();
+    }
+
+
+    private List<Integer> findZerosToTheLeftOfMaxima(double[] derivative, List<Integer> derivativeMaxima, double mean) {
+
+        List<Integer> columnsLeftOfMaxima = new ArrayList<>();
+
+        for (int i = 0; i < derivativeMaxima.size(); i++) {
+            List<Integer> consecutiveZeroes = new LinkedList<>();
+            boolean maximumFound = false;
+            int maximaIdx = derivativeMaxima.get(i) - 1; // the highest derivative will always be at least one step away from the lowest value.
+            int endIdx = (int) Math.max(globalStartIdx,
+                                        Math.min(maximaIdx - 1,
+                                                 maximaIdx - 0.1 * bins_num)); // search through 10% of array to the right, but at least one step and at most to the left edge;
+
+            for (int j = maximaIdx; j >= endIdx; j--) {
+                if (derivative[j] < DERIVATIVE_ZERO_THRESHOLD) {
+                    maximumFound = true;
+                    consecutiveZeroes.add(j);
+                } else if (maximumFound) {
+                    break;
+                }
+            }
+            if (maximumFound) {
+                int midIdx = consecutiveZeroes.size() / 2;
+                int middleMinimumIdx = consecutiveZeroes.get(midIdx);
+                if (histogram[middleMinimumIdx] < mean * MINIMUM_THRESHOLD_FOR_COLUMNS) {
+                    columnsLeftOfMaxima.add(middleMinimumIdx);
+                }
+            }
+        }
+        return columnsLeftOfMaxima;
+    }
+
+
+    private List<Integer> findZerosToTheRightOfMinima(double[] derivative, List<Integer> derivativeMinima, double mean) {
+
+        List<Integer> columnIndixes = new LinkedList<>();
+        for (int i = 0; i < derivativeMinima.size(); i++) {
+            List<Integer> consecutiveZeroes = new LinkedList<>();
+            boolean minimumFound = false;
+            int minimaIdx = derivativeMinima.get(i) + 1; // the highest derivative will always be at least one step earlier than the lowest value.
+            int endIdx = (int) Math.min(globalEndIdx,
+                                        Math.max(minimaIdx + 1,
+                                                 minimaIdx + 0.1 * bins_num)); // search through 10% of array to the right, but at least one step and at most to the right edge;
+
+            for (int j = minimaIdx; j < endIdx; j++) {
+                if (derivative[j] < DERIVATIVE_ZERO_THRESHOLD) {
+                    minimumFound = true;
+                    consecutiveZeroes.add(j);
+                } else if (minimumFound) {
+                    break;
+                }
+            }
+            if (minimumFound) {
+                int midIdx = consecutiveZeroes.size() / 2;
+                int middleMinimumIdx = consecutiveZeroes.get(midIdx);
+                if (histogram[middleMinimumIdx] < mean * MINIMUM_THRESHOLD_FOR_COLUMNS) {
+                    columnIndixes.add(middleMinimumIdx);
+                }
+            }
+        }
+        return columnIndixes;
+    }
+
+
+    private double calcMax(double[] array) {
+
+        double max = Double.NEGATIVE_INFINITY;
+        for (int i = 0; i < array.length; i++) {
+            if (array[i] > max) {
+                max = array[i];
+            }
+        }
+        return max;
+    }
+
+
+    private double calcMin(double[] array) {
+
+        double min = Double.POSITIVE_INFINITY;
+        for (int i = 0; i < array.length; i++) {
+            if (array[i] < min) {
+                min = array[i];
+            }
+        }
+        return min;
+    }
+
+
+    private Extrema calculateNearGlobalExtrema(double[] derivative, double maxDvValue, double minDvValue) {
+
+        List<Integer> nearGlobalDvMaximaIdx = new LinkedList<>();
+        List<Integer> nearGlobalDvMinimaIdx = new LinkedList<>();
+        for (int i = globalStartIdx; i < globalEndIdx; i++) {
+            if (derivative[i] <= minDvValue * NEAR_GLOBAL_THRESHOLD) {
+                nearGlobalDvMinimaIdx.add(i);
+            }
+            if (derivative[i] >= maxDvValue * NEAR_GLOBAL_THRESHOLD) {
+                nearGlobalDvMaximaIdx.add(i);
+            }
+        }
+
+        nearGlobalDvMinimaIdx = removeConsecutive(nearGlobalDvMinimaIdx);
+        nearGlobalDvMaximaIdx = removeConsecutive(nearGlobalDvMaximaIdx);
+
+        return new Extrema(nearGlobalDvMaximaIdx, nearGlobalDvMinimaIdx);
+    }
+
+
+    private record Extrema(List<Integer> maxima, List<Integer> minima) {
+
+    }
+
+
+    private Double calculateXCoordinateFromIdx(int globalMinIdx) {
+
+        return min + ((globalMinIdx + 1) * resolution);
+    }
+
+
+    public static List<Integer> removeConsecutive(List<Integer> numbers) {
+
+        List<Integer> result = new ArrayList<>();
+        if (numbers == null || numbers.isEmpty()) {
+            return result;
+        }
+
+        result.add(numbers.get(0)); // Add the first number
+
+        for (int i = 1; i < numbers.size(); i++) {
+            if (numbers.get(i) != numbers.get(i - 1) + 1) {
+                result.add(numbers.get(i)); // Add non-consecutive numbers
+            }
+        }
+
+        return result;
+    }
+
+
+    public void kernelSmooth(double[] kernel) {
+
+        double[] newFrequencies = new double[histogram.length];
+        int shift = (kernel.length - 1) / 2;
+        for (int i = 0; i < kernel.length; i++) {
+            int jStart = Math.max(0, i - shift);
+            int jEnd = Math.min(histogram.length, histogram.length + i - shift);
+            for (int j = jStart; j < jEnd; j++) {
+                newFrequencies[j - i + shift] += kernel[i] * histogram[j];
+            }
+        }
+        histogram = newFrequencies;
+    }
+
+
+    public double[] createGaussianKernel(int length, double stdDeviation) {
+
+        int r = length / 2;
+
+        int size = 2 * r + 1;
+        double[] kernel = new double[size];
+        double sum = 0;
+        double b = 2 * (stdDeviation) * (stdDeviation);
+        double a = 1 / Math.sqrt(Math.PI * b);
+        for (int i = 0; i < size; i++) {
+            kernel[i] = a * Math.exp(-(i - r) * (i - r) / b);
+            sum += kernel[i];
+        }
+        for (int i = 0; i < size; i++) {
+            kernel[i] /= sum;
+        }
+        return kernel;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Histogram.java

@@ -0,0 +1,90 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+public class Histogram {
+
+    private static final double EPSILON = 1.0e-6;
+    private final double min;
+    private final double resolution;
+    private double[] frequencies;
+
+
+    public Histogram(double minValue, double maxValue, double resolution) {
+
+        this.min = minValue - EPSILON;
+        double delta = maxValue - minValue + 2 * EPSILON;
+        int size = Math.max(1, (int) Math.round((maxValue - minValue) / resolution));
+        this.resolution = delta / size;
+        this.frequencies = new double[size];
+    }
+
+
+    public void kernelSmooth(double[] kernel) {
+
+        double[] newFrequencies = new double[frequencies.length];
+        int shift = (kernel.length - 1) / 2;
+        for (int i = 0; i < kernel.length; i++) {
+            int jStart = Math.max(0, i - shift);
+            int jEnd = Math.min(frequencies.length, frequencies.length + i - shift);
+            for (int j = jStart; j < jEnd; j++) {
+                newFrequencies[j - i + shift] += kernel[i] * frequencies[j];
+            }
+        }
+        frequencies = newFrequencies;
+    }
+
+
+    public double[] createGaussianKernel(double length, double stdDeviation) {
+
+        int r = (int) Math.round(length / resolution) / 2;
+
+        int size = 2 * r + 1;
+        double[] kernel = new double[size];
+        double sum = 0;
+        double b = 2 * (stdDeviation / resolution) * (stdDeviation / resolution);
+        double a = 1 / Math.sqrt(Math.PI * b);
+        for (int i = 0; i < size; i++) {
+            kernel[i] = a * Math.exp(-(i - r) * (i - r) / b);
+            sum += kernel[i];
+        }
+        for (int i = 0; i < size; i++) {
+            kernel[i] /= sum;
+        }
+        return kernel;
+    }
+
+
+    public void gaussianSmooth(double windowLength, double stdDeviation) {
+
+        kernelSmooth(createGaussianKernel(windowLength, stdDeviation));
+    }
+
+
+    public void add(double value) {
+
+        frequencies[(int) ((value - min) / resolution)] += 1.0;
+    }
+
+
+    public int getSize() {
+
+        return frequencies.length;
+    }
+
+
+    public double getPeakValue() {
+
+        int peakIndex = 0;
+        for (int i = 1; i < frequencies.length; i++) {
+            if (frequencies[i] > frequencies[peakIndex]) {
+                peakIndex = i;
+            }
+        }
+        int peakEndIndex = peakIndex + 1;
+        final double EPS = 0.0001;
+        while (peakEndIndex < frequencies.length && Math.abs(frequencies[peakEndIndex] - frequencies[peakIndex]) < EPS) {
+            peakEndIndex++;
+        }
+        return ((double) peakIndex + peakEndIndex) / 2 * resolution + min;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -0,0 +1,194 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.BOLD_ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.ITALIC;
+import static com.knecon.fforesight.service.layoutparser.processor.model.text.Word.STANDARD;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.FastAtan2;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.FontStyle;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Data
+@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false)
+public class Line extends TextBoundingBox {
+
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.17;
+
+    @EqualsAndHashCode.Include
+    private final double x0;
+    @EqualsAndHashCode.Include
+    private final double y0;
+
+    @EqualsAndHashCode.Include
+    private final double x1;
+    @EqualsAndHashCode.Include
+    private final double y1;
+
+    private FontStyle fontStyle;
+
+    private final List<Word> words;
+
+
+    public Line(List<Character> characters, double wordSpacing) {
+
+        if (characters.size() >= 2) {
+            // linear regression
+            double sx = 0.0;
+            double sxx = 0.0;
+            double sxy = 0.0;
+            double sy = 0.0;
+            for (Character character : characters) {
+                sx += character.getX();
+                sxx += character.getX() * character.getX();
+                sxy += character.getX() * character.getY();
+                sy += character.getY();
+            }
+            double b = (characters.size() * sxy - sx * sy) / (characters.size() * sxx - sx * sx);
+            double a = (sy - b * sx) / characters.size();
+
+            this.x0 = characters.get(0).getX();
+            this.y0 = a + b * this.x0;
+            this.x1 = characters.get(characters.size() - 1).getX();
+            this.y1 = a + b * this.x1;
+        } else {
+            Character character = characters.get(0);
+            double dx = character.getTextPosition().getWidthDirAdj() / 3;
+            double dy = dx * Math.tan(0);
+            this.x0 = character.getX() - dx;
+            this.x1 = character.getX() + dx;
+            this.y0 = character.getY() - dy;
+            this.y1 = character.getY() + dy;
+        }
+        this.words = new ArrayList<>();
+        computeWords(characters, wordSpacing * WORD_DISTANCE_MULTIPLIER);
+        buildBBox();
+        computeFontStyle();
+    }
+
+
+    public Line(List<Word> words) {
+
+        this.words = words;
+        buildBBox();
+        x0 = getMinX();
+        y0 = getMinY();
+        x1 = getMaxX();
+        y1 = getMaxY();
+        computeFontStyle();
+    }
+
+
+    private void computeFontStyle() {
+
+        EnumMap<FontStyle, AtomicInteger> fontStyleCounter = new EnumMap<>(FontStyle.class);
+        for (FontStyle fontStyle : FontStyle.values()) {
+            fontStyleCounter.put(fontStyle, new AtomicInteger(0));
+        }
+        for (Word word : words) {
+            switch (word.getFontStyle()) {
+                case STANDARD -> fontStyleCounter.get(FontStyle.REGULAR).getAndIncrement();
+                case BOLD -> fontStyleCounter.get(FontStyle.BOLD).getAndIncrement();
+                case ITALIC -> fontStyleCounter.get(FontStyle.ITALIC).getAndIncrement();
+                case BOLD_ITALIC -> fontStyleCounter.get(FontStyle.BOLD_ITALIC).getAndIncrement();
+            }
+        }
+        fontStyle = fontStyleCounter.entrySet()
+                .stream()
+                .max(Comparator.comparing(entry -> entry.getValue().get()))
+                .map(Map.Entry::getKey).orElse(FontStyle.REGULAR);
+    }
+
+
+    public double getAngle() {
+
+        return FastAtan2.fastAtan2(y1 - y0, x1 - x0);
+    }
+
+
+    public double getLength() {
+
+        return Math.sqrt((x0 - x1) * (x0 - x1) + (y0 - y1) * (y0 - y1));
+    }
+
+
+    public double angularDifference(Line j) {
+
+        double diff = Math.abs(getAngle() - j.getAngle());
+        if (diff <= Math.PI / 2) {
+            return diff;
+        } else {
+            return Math.PI - diff;
+        }
+    }
+
+
+    public double horizontalDistance(Line other) {
+
+        double[] xs = new double[4];
+        xs[0] = x0;
+        xs[1] = x1;
+        xs[2] = other.x0;
+        xs[3] = other.x1;
+        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
+        Arrays.sort(xs);
+        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
+    }
+
+
+    public double verticalDistance(Line other) {
+
+        double ym = (y0 + y1) / 2;
+        double yn = (other.y0 + other.y1) / 2;
+        return Math.abs(ym - yn);
+    }
+
+
+    private void computeWords(List<Character> characters, double wordSpacing) {
+
+        // Imo, the width of space should be scaled with the font size, but it only depends on the median distance between horizontal neighbours.
+        // If there are large differences in fontsize on a page, this might lead to missing spaces for the smaller fonts and too many for larger fonts.
+        // I've just now changed the scaling factor. If you come across this comment with missing whitespaces again, try scaling the fontsize instead of simply changing the factor again.
+        Word word = new Word();
+        Character previous = null;
+        for (Character current : characters) {
+            if (previous != null) {
+                double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
+                if (dist > wordSpacing) {
+                    words.add(word);
+                    word = new Word();
+                }
+            }
+            word.add(current);
+            previous = current;
+        }
+        words.add(word);
+    }
+
+
+    private void buildBBox() {
+
+        this.setToBBoxOfComponents(words);
+    }
+
+
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        words.forEach(word -> sb.append(word.toString()).append(" "));
+        return sb.toString().trim();
+    }
+
+}
+

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Neighbor.java

@@ -0,0 +1,43 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import lombok.Getter;
+
+public class Neighbor {
+
+    @Getter
+    private final double distance;
+    private Double angle;
+    private final Character originCharacter;
+    @Getter
+    private final Character character;
+
+
+    public Neighbor(Character neighbor, Character origin) {
+
+        this.distance = neighbor.distance(origin);
+        this.character = neighbor;
+        this.originCharacter = origin;
+    }
+
+
+    public double getHorizontalDistance() {
+
+        return character.horizontalDistance(originCharacter);
+    }
+
+
+    public double getVerticalDistance() {
+
+        return character.verticalDistance(originCharacter);
+    }
+
+
+    public double getAngle() {
+
+        if (angle != null) {
+            return angle;
+        }
+        return this.character.angle(this.originCharacter);
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/TextBoundingBox.java

@@ -0,0 +1,180 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.SuperBuilder;
+
+@Getter
+@Setter
+@SuperBuilder
+@NoArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+public abstract class TextBoundingBox extends BoundingBox {
+
+    protected Rectangle2D bBoxDirAdj;
+
+    protected TextDirection dir;
+
+
+    @Override
+    public void setToBBoxOfComponents(List<? extends BoundingBox> components) {
+
+        super.setToBBoxOfComponents(components);
+        this.bBoxDirAdj = components.stream()
+                .filter(c -> c instanceof TextBoundingBox)
+                .map(c -> (TextBoundingBox) c)
+                .map(TextBoundingBox::getBBoxDirAdj)
+                .collect(RectangleTransformations.collectBBox());
+
+        Set<TextDirection> textDirections = components.stream()
+                .filter(c -> c instanceof TextBoundingBox)
+                .map(c -> (TextBoundingBox) c)
+                .map(TextBoundingBox::getDir)
+                .collect(Collectors.toSet());
+
+        if (textDirections.isEmpty()) {
+            dir = TextDirection.ZERO;
+        } else if (textDirections.size() > 1) {
+            throw new IllegalArgumentException("More than one text direction found");
+        } else {
+            dir = textDirections.iterator().next();
+        }
+    }
+
+
+    public double getXDirAdj() {
+
+        return this.bBoxDirAdj.getX();
+    }
+
+
+    public double getYDirAdj() {
+
+        return this.bBoxDirAdj.getY();
+    }
+
+
+    public double getWidthDirAdj() {
+
+        return this.bBoxDirAdj.getWidth();
+    }
+
+
+    public double getHeightDirAdj() {
+
+        return this.bBoxDirAdj.getHeight();
+    }
+
+
+    public double getMaxXDirAdj() {
+
+        return this.bBoxDirAdj.getMaxX();
+    }
+
+
+    public double getMaxYDirAdj() {
+
+        return this.bBoxDirAdj.getMaxY();
+    }
+
+
+    public double getCenterYDirAdj() {
+
+        return this.bBoxDirAdj.getCenterY();
+    }
+
+
+    public double getCenterXDirAdj() {
+
+        return this.bBoxDirAdj.getCenterX();
+    }
+
+
+    public double horizontalDistanceDirAdj(TextBoundingBox other) {
+
+        double rect1Right = getMaxXDirAdj();
+        double rect1Left = getXDirAdj();
+        double rect2Right = other.getMaxXDirAdj();
+        double rect2Left = other.getXDirAdj();
+
+        if (rect1Left > rect2Right || rect2Left > rect1Right) {
+            return Math.max(rect2Left - rect1Right, rect1Left - rect2Right);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public double verticalDistanceDirAdj(TextBoundingBox other) {
+
+        double rect1Top = getMaxYDirAdj();
+        double rect1Bottom = getYDirAdj();
+        double rect2Top = other.getMaxYDirAdj();
+        double rect2Bottom = other.getYDirAdj();
+
+        if (rect1Bottom > rect2Top || rect2Bottom > rect1Top) {
+            return Math.max(rect2Bottom - rect1Top, rect1Bottom - rect2Top);
+        } else {
+            return 0;
+        }
+    }
+
+
+    public boolean intersectsDirAdj(TextBoundingBox other) {
+
+        return this.intersectsXDirAdj(other) && this.intersectsYDirAdj(other);
+    }
+
+
+    public boolean intersectsDirAdj(TextBoundingBox other, float yThreshold, float xThreshold) {
+
+        return this.intersectsXDirAdj(other, xThreshold) && this.intersectsYDirAdj(other, yThreshold);
+    }
+
+
+    public boolean intersectsXDirAdj(TextBoundingBox other, float threshold) {
+
+        return this.getXDirAdj() - threshold <= other.getMaxXDirAdj() && this.getMaxXDirAdj() + threshold >= other.getXDirAdj();
+    }
+
+
+    public boolean intersectsXDirAdj(TextBoundingBox other) {
+
+        return this.getXDirAdj() <= other.getMaxXDirAdj() && this.getMaxXDirAdj() >= other.getXDirAdj();
+    }
+
+
+    public boolean intersectsYDirAdj(TextBoundingBox other) {
+
+        return this.getYDirAdj() <= other.getMaxYDirAdj() && this.getMaxYDirAdj() >= other.getYDirAdj();
+    }
+
+
+    public boolean intersectsYDirAdj(TextBoundingBox other, float threshold) {
+
+        return this.getYDirAdj() - threshold <= other.getMaxYDirAdj() && this.getMaxYDirAdj() + threshold >= other.getYDirAdj();
+    }
+
+
+    public boolean isAboveDirAdj(TextBoundingBox other) {
+
+        return other.isBelow(this);
+    }
+
+
+    public boolean isBelowDirAdj(TextBoundingBox other) {
+
+        return this.intersectsXDirAdj(other) && this.getYDirAdj() >= other.getMaxYDirAdj();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/UnionFind.java

@@ -0,0 +1,37 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class UnionFind<T> extends org.jgrapht.alg.util.UnionFind<T> {
+
+    public UnionFind(Set<T> elements) {
+
+        super(elements);
+    }
+
+
+    public Collection<Set<T>> getGroups() {
+
+        Map<T, Set<T>> setRep = new LinkedHashMap<>();
+        for (T t : getParentMap().keySet()) {
+            T representative = find(t);
+            if (!setRep.containsKey(representative)) {
+                setRep.put(representative, new LinkedHashSet<>());
+            }
+            setRep.get(representative).add(t);
+        }
+
+        return setRep.values();
+    }
+
+
+    public Collection<T> getElements() {
+
+        return getParentMap().keySet();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Zone.java

@@ -0,0 +1,33 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.model;
+
+import java.util.Comparator;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Data
+@EqualsAndHashCode(callSuper = false)
+public class Zone extends TextBoundingBox {
+
+    private List<Line> lines;
+
+
+    @SuppressWarnings("PMD.ConstructorCallsOverridableMethod")
+    public Zone(List<Line> lines) {
+
+        this.lines = lines;
+        setToBBoxOfComponents(lines);
+    }
+
+
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        lines.forEach(line -> sb.append(line.toString()).append("\n"));
+        return sb.toString().trim();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/LineBuilderService.java

@@ -0,0 +1,58 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+
+@Service
+public class LineBuilderService {
+
+    private static final double CHARACTER_SPACING_DISTANCE_MULTIPLIER = 3.5;
+    private static final double LINE_SPACING_THRESHOLD_MULTIPLIER = 0.67;
+    private static final double ANGLE_TOLERANCE = Math.toRadians(5);
+
+
+    public List<Line> buildLines(List<Character> characters, double characterSpacing, double lineSpacing, CleanRulings rulings) {
+
+        double maxHorizontalDistance = characterSpacing * CHARACTER_SPACING_DISTANCE_MULTIPLIER;
+        double maxVerticalDistance = lineSpacing * LINE_SPACING_THRESHOLD_MULTIPLIER;
+
+        UnionFind<Character> unionFind = new UnionFind<>(new HashSet<>(characters));
+
+        AngleFilter angleFilter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
+
+        characters.forEach(character -> {
+            character.getNeighbors()
+                    .forEach(neighbor -> {
+                        double normalizedHorizontalDistance = neighbor.getHorizontalDistance() / maxHorizontalDistance;
+                        double normalizedVerticalDistance = neighbor.getVerticalDistance() / maxVerticalDistance;
+
+                        if (character.getTextPosition().getDir() != neighbor.getCharacter().getTextPosition().getDir() //
+                            || !angleFilter.matches(neighbor) //
+                            || Math.pow(normalizedHorizontalDistance, 2) + Math.pow(normalizedVerticalDistance, 2) > 1 //
+                            || rulings.lineBetween(character.getTextPosition(), neighbor.getCharacter().getTextPosition())) {
+                            return;
+                        }
+
+                        unionFind.union(character, neighbor.getCharacter());
+                    });
+        });
+
+        return unionFind.getGroups()
+                .stream()
+                .map(lineCharacters -> lineCharacters.stream()
+                        .sorted(Comparator.comparingDouble(Character::getX))
+                        .toList())
+                .map(lineCharacters -> new Line(lineCharacters, characterSpacing))
+                .toList();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/NearestNeighbourService.java

@@ -0,0 +1,36 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+import org.tinspin.index.Index;
+import org.tinspin.index.kdtree.KDTree;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;
+
+@Service
+public class NearestNeighbourService {
+
+    private static final int NUMBER_OF_NEIGHBOURS = 8;
+
+
+    public void findNearestNeighbors(List<Character> characters) {
+        KDTree<Character> kdTree = KDTree.create(2);
+        characters.forEach(c -> kdTree.insert(new double[]{c.getX(), c.getY()}, c));
+
+        for(Character c : characters) {
+            Index.PointIteratorKnn<Character> iterator = kdTree.queryKnn(new double[]{c.getX(), c.getY()}, NUMBER_OF_NEIGHBOURS + 1);
+            // skip the first as this is identity
+            if(iterator.hasNext()) {
+                iterator.next();
+            }
+            while(iterator.hasNext()) {
+                c.getNeighbors().add(new Neighbor(iterator.next().value(), c));
+            }
+        }
+
+    }
+
+}
+

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ReadingOrderService.java

@@ -0,0 +1,192 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
+
+@Service
+public class ReadingOrderService {
+
+    private static final double THRESHOLD = 5;
+    public static final double MULTI_COLUMN_DETECTION_THRESHOLD = 1.5;
+
+    private static final Comparator<TextBoundingBox> COMPARATOR = //
+            Comparator.comparing(TextBoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(TextBoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD));
+
+    private static final Comparator<TextBoundingBox> COMPARATOR_DIR_ADJ = //
+            Comparator.comparing(TextBoundingBox::getYDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                    .thenComparing(TextBoundingBox::getXDirAdj, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD));
+
+
+    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder, boolean useDirAdjCoords) {
+
+        if (zones.isEmpty() || zones.size() == 1) {
+            return zones;
+        }
+
+        if (xyReadingOrder) {
+            return resolveSingleColumnReadingOrder(zones, useDirAdjCoords);
+        }
+
+        Map<Long, Integer> histogram = new HashMap<>();
+        for (Zone zone : zones) {
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            long minY = Math.round(bbox.getMinY());
+            long maxY = Math.round(bbox.getMaxY());
+            for (long i = minY; i <= maxY; i++) {
+                histogram.put(i, histogram.getOrDefault(i, 0) + 1);
+            }
+        }
+
+        if (histogram.values()
+                    .stream()
+                    .mapToInt(Integer::intValue).average()
+                    .orElse(1) < MULTI_COLUMN_DETECTION_THRESHOLD) {
+            return resolveSingleColumnReadingOrder(zones, useDirAdjCoords);
+        } else {
+
+            return resolveMultiColumnReadingOder(zones, useDirAdjCoords);
+        }
+
+    }
+
+
+    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones, boolean useDirAdjCoords) {
+
+        if (useDirAdjCoords) {
+            return zones.stream()
+                    .collect(Collectors.groupingBy(TextBoundingBox::getDir)).values()
+                    .stream()
+                    .flatMap(words -> words.stream()
+                            .sorted(COMPARATOR_DIR_ADJ))
+                    .toList();
+        }
+
+        zones.sort(COMPARATOR);
+        return zones;
+    }
+
+
+    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones, boolean useDirAdjCoords) {
+
+        // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
+        // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
+
+        double minX = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+
+        for (Zone zone : zones) {
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            if (bbox.getX() < minX) {
+                minX = zone.getXDirAdj();
+            }
+            if (bbox.getMaxX() > maxX) {
+                maxX = zone.getMaxXDirAdj();
+            }
+        }
+
+        double midLineXCoordinate = (minX + maxX) / 2;
+
+        List<Zone> leftOf = new ArrayList<>();
+        List<Zone> rightOf = new ArrayList<>();
+        List<Zone> middle = new ArrayList<>();
+
+        for (Zone zone : zones) {
+            Rectangle2D bbox = useDirAdjCoords ? zone.getBBoxDirAdj() : zone.getBBox();
+            if (bbox.getX() < midLineXCoordinate && bbox.getX() + bbox.getWidth() < midLineXCoordinate) {
+                leftOf.add(zone);
+            } else if (bbox.getX() > midLineXCoordinate && bbox.getX() + bbox.getWidth() > midLineXCoordinate) {
+                rightOf.add(zone);
+            } else {
+                middle.add(zone);
+            }
+        }
+
+        if (useDirAdjCoords) {
+            leftOf.sort(COMPARATOR_DIR_ADJ);
+            rightOf.sort(COMPARATOR_DIR_ADJ);
+            middle.sort(COMPARATOR_DIR_ADJ);
+        } else {
+            leftOf.sort(COMPARATOR);
+            rightOf.sort(COMPARATOR);
+            middle.sort(COMPARATOR);
+        }
+/*
+        List<Zone> leftNotIntersecting = new ArrayList<>();
+        for (Zone leftZone : leftOf) {
+            boolean intersects = false;
+            for (Zone rightZone : rightOf) {
+                if (leftZone.intersectsY(rightZone)) {
+                    intersects = true;
+                    break;
+                }
+                // early stopping
+                if (rightZone.getBBox().getMinY() > leftZone.getBBox().getMaxY()) {
+                    break;
+                }
+            }
+            if (!intersects) {
+                leftNotIntersecting.add(leftZone);
+            }
+        }
+
+        List<Zone> rightNotIntersecting = new ArrayList<>();
+        for (Zone rightZone : rightOf) {
+            boolean intersects = false;
+            for (Zone leftZone : leftOf) {
+                if (rightZone.intersectsY(leftZone)) {
+                    intersects = true;
+                    break;
+                }
+                // early stopping
+                if (leftZone.getBBox().getMinY() > rightZone.getBBox().getMaxY()) {
+                    break;
+                }
+            }
+            if (!intersects) {
+                rightNotIntersecting.add(rightZone);
+            }
+        }
+
+        leftOf.removeAll(leftNotIntersecting);
+        rightOf.removeAll(rightNotIntersecting);
+
+        middle.addAll(leftNotIntersecting);
+        middle.addAll(rightNotIntersecting);
+*/
+        List<Zone> sortedZones = new ArrayList<>();
+        sortedZones.addAll(leftOf);
+        sortedZones.addAll(rightOf);
+
+        ListIterator<Zone> itty = middle.listIterator();
+
+        while (itty.hasNext()) {
+            Zone current = itty.next();
+            Rectangle2D bbox = useDirAdjCoords ? current.getBBoxDirAdj() : current.getBBox();
+            for (int i = 0; i < sortedZones.size(); i++) {
+                if (bbox.getY() < sortedZones.get(i).getY()) {
+                    sortedZones.add(i, current);
+                    itty.remove();
+                    break;
+                }
+            }
+        }
+
+        sortedZones.addAll(middle);
+
+        return sortedZones;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/SpacingService.java

@@ -0,0 +1,56 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Histogram;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Neighbor;
+
+@Service
+public class SpacingService {
+
+    private static final double SPACING_HISTOGRAM_RESOLUTION = 0.5;
+    private static final double SPACING_HISTOGRAM_SMOOTHING_LENGTH = 2.5;
+    private static final double SPACING_HIST_SMOOTHING_STANDARD_DEVIATION = 0.5;
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
+
+
+    public double computeCharacterSpacing(List<Character> characters) {
+
+        return computeSpacing(characters, 0);
+    }
+
+
+    public double computeLineSpacing(List<Character> characters) {
+
+        return computeSpacing(characters, Math.PI / 2);
+    }
+
+
+    private double computeSpacing(List<Character> characters, double angle) {
+
+        double maxDistance = Double.NEGATIVE_INFINITY;
+
+        for (Character character : characters) {
+            for (Neighbor neighbor : character.getNeighbors()) {
+                maxDistance = Math.max(maxDistance, neighbor.getDistance());
+            }
+        }
+        Histogram histogram = new Histogram(0, maxDistance, SPACING_HISTOGRAM_RESOLUTION);
+        AngleFilter angleFilter = new AngleFilter(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
+        for (Character character : characters) {
+            for (Neighbor neighbor : character.getNeighbors()) {
+                if (angleFilter.matches(neighbor)) {
+                    histogram.add(neighbor.getDistance());
+                }
+            }
+        }
+
+        histogram.gaussianSmooth(SPACING_HISTOGRAM_SMOOTHING_LENGTH, SPACING_HIST_SMOOTHING_STANDARD_DEVIATION);
+        return histogram.getPeakValue();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ZoneBuilderService.java

@@ -0,0 +1,126 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
+
+@Service
+public class ZoneBuilderService {
+
+    private static final double MIN_HORIZONTAL_DISTANCE_MULTIPLIER = -0.5;
+    private static final double MAX_VERTICAL_DISTANCE_MULTIPLIER = 1.2;
+
+    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -7;
+
+    private static final double MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER = 0.5;
+
+    private static final double MIN_LINE_SIZE_SCALE = 0.9;
+
+    private static final double MAX_LINE_SIZE_SCALE = 2.5;
+
+    private static final double ANGLE_TOLERANCE = Math.toRadians(5);
+
+    private static final double MAX_VERTICAL_MERGE_DISTANCE = 0.5;
+
+
+    public List<Zone> buildZones(List<Line> lines, double characterSpacing, double lineSpacing, CleanRulings rulings) {
+
+        double minHorizontalDistance = characterSpacing * MIN_HORIZONTAL_DISTANCE_MULTIPLIER;
+        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_DISTANCE_MULTIPLIER;
+        double minHorizontalMergeDistance = lineSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
+        double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;
+
+        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
+
+        double meanHeight = calculateMeanHeight(lines);
+
+        lines.forEach(outerLine -> {
+            lines.forEach(innerLine -> {
+
+                if (innerLine == outerLine //
+                    || unionFind.inSameSet(outerLine, innerLine)//
+                    || outerLine.angularDifference(innerLine) > ANGLE_TOLERANCE) {
+                    return;
+                }
+
+//                if (!innerLine.getFontStyle().equals(outerLine.getFontStyle()) //
+//                    && !outerLine.intersectsY(innerLine, -2f)) {
+//                    return;
+//                }
+
+                double horizontalScale = Math.min(outerLine.getHeightDirAdj(), innerLine.getHeightDirAdj()) / meanHeight;
+                horizontalScale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(horizontalScale, MAX_LINE_SIZE_SCALE));
+                double verticalScale = horizontalScale;
+
+//                if (innerLine.toString().endsWith(":")
+//                    || outerLine.toString().endsWith(":")
+//                    || numericalIdentifierPattern.matcher(innerLine.toString()).matches()
+//                    || numericalIdentifierPattern.matcher(outerLine.toString()).matches()) {
+//
+//                    horizontalScale *= 5;
+//                    verticalScale /= 10;
+//                }
+
+                double horizontalDistance = outerLine.horizontalDistance(innerLine) / horizontalScale;
+                double verticalDistance = outerLine.verticalDistance(innerLine) / verticalScale;
+
+                if ((!(minHorizontalDistance <= horizontalDistance) || !(verticalDistance <= maxVerticalDistance)) //
+                    && (!(minHorizontalMergeDistance <= horizontalDistance) || !(verticalDistance <= maxVerticalMergeDistance))) {
+                    return;
+                }
+
+                if (rulings.lineBetween(outerLine, innerLine)) {
+                    return;
+                }
+
+                unionFind.union(outerLine, innerLine);
+
+            });
+        });
+
+        return unionFind.getGroups()
+                .stream()
+                .map(group -> mergeLinesInZone(new ArrayList<>(group), characterSpacing, lineSpacing))
+                .toList();
+    }
+
+
+    private double calculateMeanHeight(List<Line> lines) {
+
+        double meanHeight = 0.0;
+        double weights = 0.0;
+        for (Line line : lines) {
+            double weight = line.getLength();
+            meanHeight += line.getHeightDirAdj() * weight;
+            weights += weight;
+        }
+        meanHeight /= weights;
+        return meanHeight;
+    }
+
+
+    private Zone mergeLinesInZone(List<Line> lines, double characterSpacing, double lineSpacing) {
+
+        Set<Word> words = lines.stream()
+                .map(Line::getWords)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toSet());
+        Collection<Set<Word>> groupedLines = TextPositionOperations.groupByLine(words);
+
+        List<Line> sortedLines = TextPositionOperations.sortLines(groupedLines);
+        return new Zone(sortedLines);
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/utils/DoubleUtils.java

@@ -0,0 +1,18 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.utils;
+
+public class DoubleUtils {
+
+    public static int compareDouble(double d1, double d2, double precision) {
+
+        if (Double.isNaN(d1) || Double.isNaN(d2)) {
+            return Double.compare(d1, d2);
+        }
+
+        if (Math.abs(d1 - d2) < precision) {
+            return 0;
+        }
+
+        return Double.compare(d1, d2);
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/utils/FastAtan2.java

@@ -0,0 +1,76 @@
+package com.knecon.fforesight.service.layoutparser.processor.docstrum.utils;
+
+public class FastAtan2 {
+
+    static final private int Size_Ac = 1000;
+    static final private int Size_Ar = Size_Ac + 1;
+    static final private double Pi = (float) Math.PI;
+    static final private double Pi_H = Pi / 2;
+
+    static final private double[] Atan2 = new double[Size_Ar];
+    static final private double[] Atan2_PM = new double[Size_Ar];
+    static final private double[] Atan2_MP = new double[Size_Ar];
+    static final private double[] Atan2_MM = new double[Size_Ar];
+
+    static final private double[] Atan2_R = new double[Size_Ar];
+    static final private double[] Atan2_RPM = new double[Size_Ar];
+    static final private double[] Atan2_RMP = new double[Size_Ar];
+    static final private double[] Atan2_RMM = new double[Size_Ar];
+
+    static {
+        for (int i = 0; i <= Size_Ac; i++) {
+            double d = (double) i / Size_Ac;
+            double x = 1;
+            double y = x * d;
+            double v = Math.atan2(y, x);
+            Atan2[i] = v;
+            Atan2_PM[i] = Pi - v;
+            Atan2_MP[i] = -v;
+            Atan2_MM[i] = -Pi + v;
+
+            Atan2_R[i] = Pi_H - v;
+            Atan2_RPM[i] = Pi_H + v;
+            Atan2_RMP[i] = -Pi_H + v;
+            Atan2_RMM[i] = -Pi_H - v;
+
+        }
+    }
+
+    @SuppressWarnings("ParameterAssignment")
+    static public double fastAtan2(double y, double x) {
+
+        if (y < 0) {
+            if (x < 0) {
+                //(y < x) because == (-y > -x)
+                if (y < x) {
+                    return Atan2_RMM[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_MM[(int) (y / x * Size_Ac)];
+                }
+            } else {
+                y = -y;
+                if (y > x) {
+                    return Atan2_RMP[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_MP[(int) (y / x * Size_Ac)];
+                }
+            }
+        } else {
+            if (x < 0) {
+                x = -x;
+                if (y > x) {
+                    return Atan2_RPM[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2_PM[(int) (y / x * Size_Ac)];
+                }
+            } else {
+                if (y > x) {
+                    return Atan2_R[(int) (x / y * Size_Ac)];
+                } else {
+                    return Atan2[(int) (y / x * Size_Ac)];
+                }
+            }
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/SectionNodeFactory.java

@@ -1,184 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import static java.lang.String.format;
-import static java.util.Collections.emptyList;
-import static java.util.stream.Collectors.groupingBy;
-
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.utils.TableMergingUtility;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class SectionNodeFactory {
-
-    public void addSection(GenericSemanticNode parentNode, List<AbstractPageBlock> pageBlocks, List<ClassifiedImage> images, DocumentGraphFactory.Context context) {
-
-        if (pageBlocks.isEmpty()) {
-            return;
-        }
-        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
-        Section section = Section.builder().documentTree(context.getDocumentTree()).build();
-
-        context.getSections().add(section);
-        blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));
-
-        section.setTreeId(getTreeId(parentNode, context, section));
-
-        addFirstHeadlineDirectlyToSection(pageBlocks, context, section);
-        if (containsTablesAndTextBlocks(pageBlocks)) {
-            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(section, subSectionPageBlocks, emptyList(), context));
-        } else {
-            addTablesAndParagraphsAndHeadlinesToSection(pageBlocks, context, section);
-        }
-
-        images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(section, image, context));
-    }
-
-
-    private List<Integer> getTreeId(GenericSemanticNode parentNode, DocumentGraphFactory.Context context, Section section) {
-
-        if (parentNode == null) {
-            return context.getDocumentTree().createNewMainEntryAndReturnId(section);
-        } else {
-            return context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, section);
-        }
-    }
-
-
-    private void addFirstHeadlineDirectlyToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
-
-        if (pageBlocks.get(0).isHeadline()) {
-            addTablesAndParagraphsAndHeadlinesToSection(List.of(pageBlocks.get(0)), context, section);
-            pageBlocks.remove(0);
-        }
-    }
-
-
-    private void addTablesAndParagraphsAndHeadlinesToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
-
-        Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
-        List<AbstractPageBlock> remainingBlocks = new LinkedList<>(pageBlocks);
-        for (AbstractPageBlock abstractPageBlock : pageBlocks) {
-
-            if (alreadyMerged.contains(abstractPageBlock)) {
-                continue;
-            }
-
-            remainingBlocks.removeAll(alreadyMerged);
-
-            if (abstractPageBlock instanceof TextPageBlock) {
-                List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY((TextPageBlock) abstractPageBlock, remainingBlocks);
-                alreadyMerged.addAll(textBlocks);
-                DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
-            } else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
-                List<TablePageBlock> tablesToMerge = TableMergingUtility.findConsecutiveTablesWithSameColCountAndSameHeaders(tablePageBlock, remainingBlocks);
-                alreadyMerged.addAll(tablesToMerge);
-                TableNodeFactory.addTable(section, tablesToMerge, context);
-            } else {
-                throw new RuntimeException(format("Unhandled AbstractPageBlockType %s!", abstractPageBlock.getClass()));
-            }
-        }
-    }
-
-
-    private boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {
-
-        return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
-    }
-
-
-    /**
-     * This function splits the list of PageBlocks around TablePageBlocks, such that SubSections can be created, that don't include tables.
-     * This is needed so we can execute rules on sections, that do not contain tables.
-     * See: <a href="https://knecon.atlassian.net/wiki/spaces/RED/pages/14765218/Document+Structure">document structure wiki</a>
-     *
-     * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock
-     * @return List of Lists of AbstractPageBlocks, which include either a single Headline ClassificationTextBlock and a TablePageBlock or only ClassificationTextBlocks.
-     */
-    private List<List<AbstractPageBlock>> splitPageBlocksIntoSubSections(List<AbstractPageBlock> pageBlocks) {
-
-        List<List<AbstractPageBlock>> splitList = splitIntoCoherentList(pageBlocks);
-        movePrecedingHeadlineToTableList(splitList);
-        return splitList.stream().filter(list -> !list.isEmpty()).toList();
-    }
-
-
-    private void movePrecedingHeadlineToTableList(List<List<AbstractPageBlock>> splitList) {
-
-        for (int i = 0; i < splitList.size(); i++) {
-            if (listIsTablesOnly(splitList.get(i)) && i > 0) {
-                List<AbstractPageBlock> previousList = splitList.get(i - 1);
-                AbstractPageBlock lastPageBlockInPreviousList = previousList.get(previousList.size() - 1);
-                if (lastPageBlockInPreviousList.isHeadline()) {
-                    previousList.remove(previousList.size() - 1);
-                    splitList.get(i).add(0, lastPageBlockInPreviousList);
-                }
-            }
-        }
-    }
-
-
-    private boolean listIsTablesOnly(List<AbstractPageBlock> abstractPageBlocks) {
-
-        return abstractPageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock);
-    }
-
-
-    /**
-     * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock
-     * @return List of Lists of AbstractPageBlocks, which are exclusively of type ClassificationTextBlock or TablePageBlock
-     */
-    private List<List<AbstractPageBlock>> splitIntoCoherentList(List<AbstractPageBlock> pageBlocks) {
-
-        List<List<AbstractPageBlock>> splitList = new LinkedList<>();
-        List<AbstractPageBlock> currentList = new LinkedList<>();
-        splitList.add(currentList);
-
-        Class<? extends AbstractPageBlock> lastPageBlockClass = pageBlocks.get(0).getClass();
-        for (AbstractPageBlock pageBlock : pageBlocks) {
-            if (lastPageBlockClass.isInstance(pageBlock)) {
-                currentList.add(pageBlock);
-            } else {
-                currentList = new LinkedList<>();
-                currentList.add(pageBlock);
-                splitList.add(currentList);
-                lastPageBlockClass = pageBlock.getClass();
-            }
-        }
-        return splitList;
-    }
-
-
-    private List<TextPageBlock> findTextBlocksWithSameClassificationAndAlignsY(TextPageBlock atc, List<AbstractPageBlock> pageBlocks) {
-
-        return pageBlocks.stream()
-                .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
-                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
-                .filter(abstractTextContainer -> abstractTextContainer instanceof TextPageBlock)
-                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
-                .map(abstractTextContainer -> (TextPageBlock) abstractTextContainer)
-                .filter(abstractTextContainer -> abstractTextContainer.getDir() == atc.getDir())
-                .toList();
-    }
-
-
-    private void addSectionNodeToPageNode(DocumentGraphFactory.Context context, Section section, Integer pageNumber) {
-
-        Page page = context.getPage(pageNumber);
-        page.getMainBody().add(section);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/TextBlockFactory.java

@@ -1,74 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
-
-import lombok.AccessLevel;
-import lombok.experimental.FieldDefaults;
-
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class TextBlockFactory {
-
-    int stringOffset;
-    long textBlockIdx;
-
-
-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
-
-        Integer numberOnPage = context.getAndIncrementTextBlockNumberOnPage(page);
-        return buildAtomicTextBlock(sequences, parent, numberOnPage, page);
-    }
-
-
-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, Integer numberOnPage, Page page) {
-
-        SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionDto(sequences);
-        int offset = stringOffset;
-        stringOffset += searchTextWithTextPositionDto.getSearchText().length();
-        long idx = textBlockIdx;
-        textBlockIdx++;
-        String orientation;
-        int textDirection;
-        if (sequences.isEmpty()) {
-            orientation = null;
-            textDirection = 0;
-        } else {
-            orientation = sequences.get(0).getDir().toString();
-            textDirection = sequences.get(0).getRotation();
-        }
-        return AtomicTextBlock.fromSearchTextWithTextPosition(searchTextWithTextPositionDto.getSearchText(),
-                searchTextWithTextPositionDto.getLineBreaks(),
-                searchTextWithTextPositionDto.getBoldTextBoundaries(),
-                searchTextWithTextPositionDto.getItalicTextBoundaries(),
-                searchTextWithTextPositionDto.getPositions(),
-                searchTextWithTextPositionDto.getStringIdxToPositionIdx(),
-                idx,
-                parent,
-                numberOnPage,
-                page,
-                offset,
-                orientation,
-                textDirection);
-    }
-
-
-    public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
-
-        long idx = textBlockIdx;
-        textBlockIdx++;
-        return AtomicTextBlock.empty(idx, stringOffset, page, context.getAndIncrementTextBlockNumberOnPage(page), parent);
-    }
-
-
-    public AtomicTextBlock emptyTextBlock(SemanticNode parent, Integer numberOnPage, Page page) {
-
-        long idx = textBlockIdx;
-        textBlockIdx++;
-        return AtomicTextBlock.empty(idx, stringOffset, page, numberOnPage, parent);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/Boundary.java

@@ -1,144 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph;
-
-import static java.lang.String.format;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.stream.IntStream;
-
-import lombok.EqualsAndHashCode;
-import lombok.Setter;
-
-@Setter
-@EqualsAndHashCode
-public class Boundary implements Comparable<Boundary> {
-
-    private int start;
-    private int end;
-
-
-    public Boundary(int start, int end) {
-
-        if (start > end) {
-            throw new IllegalArgumentException(format("start: %d > end: %d", start, end));
-        }
-        this.start = start;
-        this.end = end;
-    }
-
-
-    public int length() {
-
-        return end - start;
-    }
-
-
-    public int start() {
-
-        return start;
-    }
-
-
-    public int end() {
-
-        return end;
-    }
-
-
-    public boolean contains(Boundary boundary) {
-
-        return start <= boundary.start() && boundary.end() <= end;
-    }
-
-
-    public boolean containedBy(Boundary boundary) {
-
-        return boundary.contains(this);
-    }
-
-
-    public boolean contains(int start, int end) {
-
-        if (start > end) {
-            throw new IllegalArgumentException(format("start: %d > end: %d", start, end));
-        }
-        return this.start <= start && end <= this.end;
-    }
-
-
-    public boolean containedBy(int start, int end) {
-
-        if (start > end) {
-            throw new IllegalArgumentException(format("start: %d > end: %d", start, end));
-        }
-        return start <= this.start && this.end <= end;
-    }
-
-
-    public boolean contains(int index) {
-
-        return start <= index && index < end;
-    }
-
-
-    public boolean intersects(Boundary boundary) {
-
-        return boundary.start() < this.end && this.start < boundary.end();
-    }
-
-
-    public List<Boundary> split(List<Integer> splitIndices) {
-
-        if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
-            throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this));
-        }
-        List<Boundary> splitBoundaries = new LinkedList<>();
-        int previousIndex = start;
-        for (int splitIndex : splitIndices) {
-
-            // skip split if it would produce a boundary of length 0
-            if (splitIndex == previousIndex) {
-                continue;
-            }
-            splitBoundaries.add(new Boundary(previousIndex, splitIndex));
-            previousIndex = splitIndex;
-        }
-        splitBoundaries.add(new Boundary(previousIndex, end));
-        return splitBoundaries;
-    }
-
-    public IntStream intStream() {
-
-        return IntStream.range(start, end);
-    }
-
-    public static Boundary merge(Collection<Boundary> boundaries) {
-
-        int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
-        int maxEnd = boundaries.stream().mapToInt(Boundary::end).max().orElseThrow(IllegalArgumentException::new);
-        return new Boundary(minStart, maxEnd);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return format("Boundary [%d|%d)", start, end);
-    }
-
-
-    @Override
-    public int compareTo(Boundary boundary) {
-
-        if (end < boundary.end() && start < boundary.start()) {
-            return -1;
-        }
-        if (start > boundary.start() && end > boundary.end()) {
-            return 1;
-        }
-
-        return 0;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/Document.java

@@ -1,120 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.amazonaws.services.kms.model.NotFoundException;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Document implements GenericSemanticNode {
-
-    Set<Page> pages;
-    DocumentTree documentTree;
-    Integer numberOfPages;
-    TextBlock textBlock;
-    @Builder.Default
-    Set<RedactionEntity> entities = new HashSet<>();
-
-
-    @Override
-    public NodeType getType() {
-
-        return NodeType.DOCUMENT;
-    }
-
-
-    public TextBlock getTextBlock() {
-
-        if (textBlock == null) {
-            textBlock = streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
-        }
-        return textBlock;
-    }
-
-
-    public List<Section> getMainSections() {
-
-        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node).collect(Collectors.toList());
-    }
-
-
-    public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
-
-        return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock);
-    }
-
-
-    @Override
-    public List<Integer> getTreeId() {
-
-        return Collections.emptyList();
-    }
-
-
-    @Override
-    public void setTreeId(List<Integer> tocId) {
-
-        throw new UnsupportedOperationException("Document is always the root of the TablePageBlock of Contents");
-    }
-
-
-    @Override
-    public Headline getHeadline() {
-
-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
-    }
-
-
-    private Stream<SemanticNode> streamAllNodes() {
-
-        return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode);
-    }
-
-
-    public Stream<Image> streamAllImages() {
-
-        return streamAllSubNodesOfType(NodeType.IMAGE).map(node -> (Image) node);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return NodeType.DOCUMENT + ": " + this.getTextBlock().buildSummary();
-    }
-
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        Map<Page, Rectangle2D> bBox = new HashMap<>();
-        for (Page page : pages) {
-            bBox.put(page, new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight()));
-        }
-        return bBox;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/Footer.java

@@ -1,65 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Footer implements GenericSemanticNode {
-
-    List<Integer> treeId;
-    TextBlock leafTextBlock;
-
-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-
-    @Override
-    public NodeType getType() {
-
-        return NodeType.FOOTER;
-    }
-
-
-    @Override
-    public boolean isLeaf() {
-
-        return true;
-    }
-
-
-    @Override
-    public TextBlock getTextBlock() {
-
-        return leafTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/GenericSemanticNode.java

@@ -1,5 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-public interface GenericSemanticNode extends SemanticNode {
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/Header.java

@@ -1,65 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Header implements GenericSemanticNode {
-
-    List<Integer> treeId;
-    TextBlock leafTextBlock;
-
-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-
-    @Override
-    public boolean isLeaf() {
-
-        return true;
-    }
-
-
-    @Override
-    public NodeType getType() {
-
-        return NodeType.HEADER;
-    }
-
-
-    @Override
-    public TextBlock getTextBlock() {
-
-        return leafTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/Headline.java

@@ -1,72 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Headline implements GenericSemanticNode {
-
-    List<Integer> treeId;
-    TextBlock leafTextBlock;
-
-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-
-    @Override
-    public NodeType getType() {
-
-        return NodeType.HEADLINE;
-    }
-
-
-    @Override
-    public boolean isLeaf() {
-
-        return true;
-    }
-
-
-    @Override
-    public TextBlock getTextBlock() {
-
-        return leafTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return treeId + ": " + NodeType.HEADLINE + ": " + leafTextBlock.buildSummary();
-    }
-
-
-    @Override
-    public Headline getHeadline() {
-
-        return this;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/Paragraph.java

@@ -1,63 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Paragraph implements GenericSemanticNode {
-
-    List<Integer> treeId;
-    TextBlock leafTextBlock;
-
-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-
-    @Override
-    public NodeType getType() {
-
-        return NodeType.PARAGRAPH;
-    }
-
-
-    @Override
-    public boolean isLeaf() {
-
-        return true;
-    }
-
-
-    @Override
-    public TextBlock getTextBlock() {
-
-        return leafTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/Section.java

@@ -1,77 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Section implements GenericSemanticNode {
-
-    List<Integer> treeId;
-
-    TextBlock textBlock;
-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-
-    @Override
-    public NodeType getType() {
-
-        return NodeType.SECTION;
-    }
-
-
-    public boolean hasTables() {
-
-        return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent();
-    }
-
-
-    @Override
-    public TextBlock getTextBlock() {
-
-        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
-        }
-        return textBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return treeId.toString() + ": " + NodeType.SECTION + ": " + this.getTextBlock().buildSummary();
-    }
-
-
-    public Headline getHeadline() {
-
-        return streamChildrenOfType(NodeType.HEADLINE)//
-                .map(node -> (Headline) node)//
-                .findFirst()//
-                .orElseGet(() -> getParent().getHeadline());
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/textblock/TextBlock.java

@@ -1,148 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
-
-import static java.lang.String.format;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-
-public interface TextBlock extends CharSequence {
-
-    String getSearchText();
-
-
-    List<AtomicTextBlock> getAtomicTextBlocks();
-
-
-    List<Boundary> getBoldTextBoundaries();
-
-
-    List<Boundary> getItalicTextBoundaries();
-
-
-    String getOrientation();
-
-
-    int getTextDirection();
-
-
-    Boundary getBoundary();
-
-
-    int getNextLinebreak(int fromIndex);
-
-
-    int getPreviousLinebreak(int fromIndex);
-
-
-    List<Integer> getLineBreaks();
-
-
-    Rectangle2D getPosition(int stringIdx);
-
-
-    List<Rectangle2D> getPositions(Boundary stringBoundary);
-
-
-    Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary);
-
-
-    int numberOfLines();
-
-
-    default int indexOf(String searchTerm) {
-
-        return indexOf(searchTerm, getBoundary().start());
-    }
-
-
-    default Set<Page> getPages() {
-
-        return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet());
-    }
-
-
-    default Set<Page> getPages(Boundary boundary) {
-
-        return getAtomicTextBlocks().stream()
-                .filter(atomicTextBlock -> atomicTextBlock.getBoundary().intersects(boundary))
-                .map(AtomicTextBlock::getPage)
-                .collect(Collectors.toUnmodifiableSet());
-    }
-
-
-    default int indexOf(String searchTerm, int startOffset) {
-
-        int start = getSearchText().indexOf(searchTerm, startOffset - getBoundary().start());
-        if (start == -1) {
-            return -1;
-        }
-        return start + getBoundary().start();
-    }
-
-
-    default CharSequence getFirstLine() {
-
-        return subSequence(getBoundary().start(), getNextLinebreak(getBoundary().start()));
-    }
-
-
-    default boolean containsBoundary(Boundary boundary) {
-
-        if (boundary.end() < boundary.start()) {
-            throw new IllegalArgumentException(format("Invalid %s, StartIndex must be smaller than EndIndex", boundary));
-        }
-        return getBoundary().contains(boundary);
-    }
-
-
-    default boolean containsIndex(int stringIndex) {
-
-        return getBoundary().contains(stringIndex);
-    }
-
-
-    default CharSequence subSequence(Boundary boundary) {
-
-        return subSequence(boundary.start(), boundary.end());
-    }
-
-
-    default String buildSummary() {
-
-        String[] words = getSearchText().split(" ");
-        int bound = Math.min(words.length, 4);
-        List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
-
-        return String.join(" ", list);
-    }
-
-
-    @Override
-    default CharSequence subSequence(int start, int end) {
-
-        return getSearchText().substring(start - getBoundary().start(), end - getBoundary().start());
-    }
-
-
-    @Override
-    default int length() {
-
-        return getBoundary().length();
-    }
-
-
-    @Override
-    default char charAt(int index) {
-
-        return getSearchText().charAt(index - getBoundary().start());
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/mapper/redaction/DocumentDataMapper.java

@@ -1,147 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class DocumentDataMapper {
-
-    public DocumentData toDocumentData(Document document) {
-
-        List<DocumentTextData> documentTextData = document.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
-                .distinct()
-                .map(DocumentDataMapper::toAtomicTextBlockData)
-                .toList();
-
-        List<DocumentPositionData> atomicPositionBlockData = document.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
-                .distinct()
-                .map(DocumentDataMapper::toAtomicPositionBlockData)
-                .toList();
-
-        Set<Long> nonEmptyTextBlocks = documentTextData.stream().mapToLong(DocumentTextData::getId).boxed().collect(Collectors.toSet());
-
-        List<DocumentPage> documentPageData = document.getPages().stream().map(DocumentDataMapper::toPageData).toList();
-        DocumentStructure tableOfContentsData = toDocumentTreeData(document.getDocumentTree());
-        return DocumentData.builder()
-                .documentTextData(documentTextData.toArray(new DocumentTextData[0]))
-                .documentPositions(atomicPositionBlockData.toArray(new DocumentPositionData[0]))
-                .documentPages(documentPageData.toArray(new DocumentPage[0]))
-                .documentStructure(tableOfContentsData)
-                .build();
-    }
-
-
-    private DocumentStructure toDocumentTreeData(DocumentTree documentTree) {
-
-        return new DocumentStructure(toEntryData(documentTree.getRoot()));
-    }
-
-
-    private DocumentStructure.EntryData toEntryData(DocumentTree.Entry entry) {
-
-        Long[] atomicTextBlocks;
-
-        if (entry.getNode().isLeaf()) {
-            atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getLeafTextBlock());
-        } else {
-            atomicTextBlocks = new Long[]{};
-        }
-
-        Map<String, String> properties = switch (entry.getType()) {
-            case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode());
-            case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode());
-            case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode());
-            default -> new HashMap<>();
-        };
-
-        return DocumentStructure.EntryData.builder()
-                .treeId(toPrimitiveIntArray(entry.getTreeId()))
-                .children(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
-                .type(entry.getType())
-                .atomicBlockIds(atomicTextBlocks)
-                .pageNumbers(entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new))
-                .properties(properties)
-                .build();
-    }
-
-
-    private Long[] toAtomicTextBlockIds(TextBlock textBlock) {
-
-        return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);
-    }
-
-
-    private DocumentPage toPageData(Page p) {
-
-        return DocumentPage.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
-    }
-
-
-    private DocumentTextData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {
-
-        return DocumentTextData.builder()
-                .id(atomicTextBlock.getId())
-                .page(atomicTextBlock.getPage().getNumber().longValue())
-                .searchText(atomicTextBlock.getSearchText())
-                .numberOnPage(atomicTextBlock.getNumberOnPage())
-                .start(atomicTextBlock.getBoundary().start())
-                .end(atomicTextBlock.getBoundary().end())
-                .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
-                .build();
-    }
-
-
-    private DocumentPositionData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {
-
-        return DocumentPositionData.builder()
-                .id(atomicTextBlock.getId())
-                .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
-                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
-                .build();
-    }
-
-
-    private static float[][] toPrimitiveFloatMatrix(List<Rectangle2D> positions) {
-
-        float[][] positionMatrix = new float[positions.size()][];
-        for (int i = 0; i < positions.size(); i++) {
-            positionMatrix[i] = toArray(positions.get(i));
-        }
-        return positionMatrix;
-    }
-
-
-    private static float[] toArray(Rectangle2D positions) {
-
-        return new float[]{(float) positions.getMinX(), (float) positions.getMinY(), (float) positions.getWidth(), (float) positions.getHeight()};
-    }
-
-
-    private int[] toPrimitiveIntArray(List<Integer> list) {
-
-        return list.stream().mapToInt(Integer::intValue).toArray();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/mapper/redaction/DocumentGraphMapper.java

@@ -1,198 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Footer;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Header;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Headline;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Paragraph;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class DocumentGraphMapper {
-
-    public Document toDocumentGraph(DocumentData documentData) {
-
-        Document document = new Document();
-        DocumentTree documentTree = new DocumentTree(document);
-        Context context = new Context(documentData, documentTree);
-
-        context.pages.addAll(Arrays.stream(documentData.getDocumentPages()).map(DocumentGraphMapper::buildPage).toList());
-
-        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
-
-        document.setDocumentTree(context.documentTree);
-        document.setPages(new HashSet<>(context.pages));
-        document.setNumberOfPages(documentData.getDocumentPages().length);
-
-        document.setTextBlock(document.getTextBlock());
-        return document;
-    }
-
-
-    private List<DocumentTree.Entry> buildEntries(List<DocumentStructure.EntryData> entries, Context context) {
-
-        List<DocumentTree.Entry> newEntries = new LinkedList<>();
-        for (DocumentStructure.EntryData entryData : entries) {
-
-            List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();
-
-            SemanticNode node = switch (entryData.getType()) {
-                case SECTION -> buildSection(context);
-                case PARAGRAPH -> buildParagraph(context);
-                case HEADLINE -> buildHeadline(context);
-                case HEADER -> buildHeader(context);
-                case FOOTER -> buildFooter(context);
-                case TABLE -> buildTable(context, entryData.getProperties());
-                case TABLE_CELL -> buildTableCell(context, entryData.getProperties());
-                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers());
-                default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
-            };
-
-            if (entryData.getAtomicBlockIds().length > 0) {
-                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
-                node.setLeafTextBlock(textBlock);
-            }
-            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
-            node.setTreeId(treeId);
-
-            switch (entryData.getType()) {
-                case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
-                case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
-                default -> pages.forEach(page -> page.getMainBody().add(node));
-            }
-
-            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
-        }
-        return newEntries;
-    }
-
-
-    private Headline buildHeadline(Context context) {
-
-        return Headline.builder().documentTree(context.documentTree).build();
-    }
-
-
-    private Image buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) {
-
-        assert pageNumbers.length == 1;
-        Page page = getPage(pageNumbers[0], context);
-        var builder = Image.builder();
-        PropertiesMapper.parseImageProperties(properties, builder);
-        return builder.documentTree(context.documentTree).page(page).build();
-    }
-
-
-    private TableCell buildTableCell(Context context, Map<String, String> properties) {
-
-        TableCell.TableCellBuilder builder = TableCell.builder();
-        PropertiesMapper.parseTableCellProperties(properties, builder);
-        return builder.documentTree(context.documentTree).build();
-    }
-
-
-    private Table buildTable(Context context, Map<String, String> properties) {
-
-        Table.TableBuilder builder = Table.builder();
-        PropertiesMapper.parseTableProperties(properties, builder);
-        return builder.documentTree(context.documentTree).build();
-    }
-
-
-    private Footer buildFooter(Context context) {
-
-        return Footer.builder().documentTree(context.documentTree).build();
-    }
-
-
-    private Header buildHeader(Context context) {
-
-        return Header.builder().documentTree(context.documentTree).build();
-    }
-
-
-    private Section buildSection(Context context) {
-
-        return Section.builder().documentTree(context.documentTree).build();
-    }
-
-
-    private Paragraph buildParagraph(Context context) {
-
-        return Paragraph.builder().documentTree(context.documentTree).build();
-    }
-
-
-    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
-
-        return Arrays.stream(atomicTextBlockIds).map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)).collect(new TextBlockCollector());
-    }
-
-
-    private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
-
-        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                parent,
-                getPage(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
-    }
-
-
-    private Page buildPage(DocumentPage p) {
-
-        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
-    }
-
-
-    private Page getPage(Long pageIndex, Context context) {
-
-        return context.pages.stream()
-                .filter(page -> page.getNumber() == Math.toIntExact(pageIndex))
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException(String.format("ClassificationPage with number %d not found", pageIndex)));
-    }
-
-
-    static final class Context {
-
-        private final DocumentTree documentTree;
-        private final List<Page> pages;
-        private final List<DocumentTextData> documentTextDataBlockData;
-        private final List<DocumentPositionData> atomicPositionBlockData;
-
-
-        Context(DocumentData documentData, DocumentTree documentTree) {
-
-            this.documentTree = documentTree;
-            this.pages = new LinkedList<>();
-            this.documentTextDataBlockData = Arrays.stream(documentData.getDocumentTextData()).toList();
-            this.atomicPositionBlockData = Arrays.stream(documentData.getDocumentPositions()).toList();
-
-        }
-
-    }
-
-}