Merge branch 'RED-8481-fix' into 'main'

Red 8481 fix

See merge request fforesight/layout-parser!103

.gitignore

@@ -18,6 +18,7 @@ target/
 .settings
 .springBeans
 .sts4-cache
+.gradle
 
 ### IntelliJ IDEA ###
 .idea
@@ -37,3 +38,7 @@ build/
 
 ### VS Code ###
 .vscode/
+gradlew.bat
+gradlew
+gradle.properties
+gradle/

.gitlab-ci.yml

@@ -1,6 +1,21 @@
-variables:
-  SONAR_PROJECT_KEY: 'fforesight_layout-parser_AYd5quv2mRkBOCG22hvF'
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'
-    file: 'ci-templates/maven_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG

.mvn/wrapper/maven-wrapper.properties

@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.7/apache-maven-3.8.7-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar

README.md

@@ -1 +1,89 @@
+# PDF Layout Parser Micro-Service: layout-parser  
 
+## Introduction
+The layout-parser micro-service is a powerful tool designed to efficiently extract structured information from PDF documents. Written in Java and utilizing Spring Boot 3, Apache PDFBox, and RabbitMQ, this micro-service excels at parsing PDFs and organizing their content into a meaningful and coherent layout structure. Notably, the layout-parser micro-service distinguishes itself by relying solely on advanced algorithms, rather than machine learning techniques.
+
+### Key Steps in the PDF Layout Parsing Process:
+
+* **Text Position Extraction:**  
+The micro-service leverages Apache PDFBox to extract precise text positions for each individual character within the PDF document.
+
+* **Word Segmentation and Text Block Formation:**  
+Employing an array of diverse algorithms, the micro-service initially identifies and segments words, creating distinct text blocks.
+
+* **Text Block Classification:**  
+The segmented text blocks are then subjected to classification algorithms. These algorithms categorize the text blocks based on their content and visual properties, distinguishing between sections, subsections, headlines, paragraphs, images, tables, table cells, headers, and footers.
+
+* **Layout Coherence Establishment:**  
+The classified text blocks are subsequently orchestrated into a cohesive layout structure. This process involves arranging sections, subsections, paragraphs, images, and other elements in a logical and structured manner.
+
+* **Output Generation in Various Formats:**  
+Once the layout structure is established, the micro-service generates output in multiple formats. These formats are designed for seamless integration with downstream micro-services. The supported formats include JSON, XML, and others, ensuring flexibility in downstream data consumption.
+
+### Optional Enhancements:
+
+* **ML-Based Table Extraction:**  
+For enhanced results, users have the option to incorporate machine learning-based table extraction. This feature can be activated by providing ML-generated results as a JSON file, which are then integrated seamlessly into the layout structure.
+
+* **Image Classification using ML:**  
+Additionally, for more accurate image classification, users can optionally feed ML-generated image classification results into the micro-service. Similar to the table extraction option, the micro-service processes the pre-parsed results in JSON format, thus optimizing the accuracy of image content identification.
+
+In conclusion, the layout-parser micro-service is a versatile PDF layout parsing solution crafted entirely around advanced algorithms, without reliance on machine learning. It proficiently extracts text positions, segments content into meaningful blocks, classifies these blocks, arranges them coherently, and outputs structured data for downstream micro-services. Optional integration with ML-generated table extractions and image classifications further enhances its capabilities.
+
+
+
+
+
+
+## Installation
+
+### Prerequisites
+
+Before building and using the layout-parser micro-service, please ensure you have the following software and tools installed:
+
+Java Development Kit (JDK) 17 or later
+Gradle build tool (preinstalled)
+Build and Test
+To build and test the micro-service, follow these steps:
+
+### Clone the Repository:
+
+bash
+```
+git clone ssh://git@git.knecon.com:22222/fforesight/layout-parser.git
+cd layout-parser
+```
+### Build the Project:
+Use the following command to build the project using Gradle:
+
+```
+gradle clean build
+```
+### Run Tests:
+Run the test suite using the following command:
+```
+gradle test
+```
+## Building a Custom Docker Image
+To create a custom Docker image for the layout-parser micro-service, execute the provided script:
+
+### Ensure Docker is Installed:
+Ensure that Docker is installed and running on your system.
+
+### Run the Image Building Script:
+Execute the publish-custom-image script in the project directory:
+
+```
+./publish-custom-image
+```
+## Publishing to Internal Maven Repository
+To publish the layout-parser micro-service to your internal Maven repository, execute the following command:
+
+```
+gradle -Pversion=buildVersion publish
+```
+Replace buildVersion with the desired version number.
+
+## Additional Notes
+Make sure to configure any necessary application properties before deploying the micro-service.
+For advanced usage and configurations, refer to Kilian or Dom or preferably the source code.

buildSrc/build.gradle.kts

@@ -0,0 +1,7 @@
+plugins {
+    `kotlin-dsl`
+}
+
+repositories {
+    gradlePluginPortal()
+}

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -0,0 +1,77 @@
+plugins {
+    `java-library`
+    `maven-publish`
+    pmd
+    checkstyle
+    jacoco
+}
+
+group = "com.knecon.fforesight"
+
+java.sourceCompatibility = JavaVersion.VERSION_17
+java.targetCompatibility = JavaVersion.VERSION_17
+
+tasks.pmdMain {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
+}
+
+tasks.pmdTest {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/test_pmd.xml")
+}
+
+tasks.named<Test>("test") {
+    useJUnitPlatform()
+    reports {
+        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
+    }
+    minHeapSize = "512m"
+    maxHeapSize = "2048m"
+}
+
+tasks.test {
+    finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
+}
+
+tasks.jacocoTestReport {
+    dependsOn(tasks.test) // tests are required to run before generating the report
+    reports {
+        xml.required.set(true)
+        csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
+    }
+}
+
+allprojects {
+    publishing {
+        publications {
+            create<MavenPublication>(name) {
+                from(components["java"])
+            }
+        }
+        repositories {
+            maven {
+                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+                credentials {
+                    username = providers.gradleProperty("mavenUser").getOrNull();
+                    password = providers.gradleProperty("mavenPassword").getOrNull();
+                }
+            }
+        }
+    }
+}
+
+java {
+    withJavadocJar()
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven {
+        url = uri("https://nexus.knecon.com/repository/gindev/")
+        credentials {
+            username = providers.gradleProperty("mavenUser").getOrNull();
+            password = providers.gradleProperty("mavenPassword").getOrNull();
+        }
+    }
+}

config/checkstyle/checkstyle.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <property
+            name="severity"
+            value="error"/>
+    <module name="TreeWalker">
+        <module name="SuppressWarningsHolder"/>
+        <module name="MissingDeprecated"/>
+        <module name="MissingOverride"/>
+        <module name="AnnotationLocation"/>
+        <module name="JavadocStyle"/>
+        <module name="NonEmptyAtclauseDescription"/>
+        <module name="IllegalImport"/>
+        <module name="RedundantImport"/>
+        <module name="RedundantModifier"/>
+        <module name="EmptyBlock"/>
+        <module name="DefaultComesLast"/>
+        <module name="EmptyStatement"/>
+        <module name="EqualsHashCode"/>
+        <module name="ExplicitInitialization"/>
+        <module name="IllegalInstantiation"/>
+        <module name="ModifiedControlVariable"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="PackageDeclaration"/>
+        <module name="ParameterAssignment"/>
+        <module name="SimplifyBooleanExpression"/>
+        <module name="SimplifyBooleanReturn"/>
+        <module name="StringLiteralEquality"/>
+        <module name="OneStatementPerLine"/>
+        <module name="FinalClass"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="UpperEll"/>
+        <module name="OuterTypeFilename"/>
+    </module>
+    <module name="FileTabCharacter"/>
+    <module name="SuppressWarningsFilter"/>
+</module>

config/pmd/pmd.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon ruleset checks the code for bad stuff
+    </description>
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AssignmentInOperand"/>
+        <exclude name="BeanMembersShouldSerialize"/>
+    </rule>
+
+</ruleset>
+

config/pmd/test_pmd.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon test ruleset checks the code for bad stuff
+    </description>
+
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AssignmentInOperand"/>
+        <exclude name="TestClassWithoutTestCases"/>
+        <exclude name="BeanMembersShouldSerialize"/>
+    </rule>
+
+</ruleset>

gradle.properties.kts

@@ -0,0 +1 @@
+version = 0.1-SNAPSHOT

layoutparser-service-image/pom.xml

@@ -1,99 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-
-
-    <parent>
-        <groupId>com.iqser.red</groupId>
-        <artifactId>platform-docker-dependency</artifactId>
-        <version>1.2.0</version>
-        <relativePath/>
-    </parent>
-
-    <groupId>com.knecon.fforesight</groupId>
-    <artifactId>layoutparser-service-image</artifactId>
-    <version>1.0.0</version>
-    <packaging>pom</packaging>
-
-    <properties>
-        <service.server>layoutparser-service-server</service.server>
-        <platform.jar>${service.server}.jar</platform.jar>
-        <docker.skip.push>false</docker.skip.push>
-        <docker.image.name>${docker.image.prefix}/${service.server}</docker.image.name>
-    </properties>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-dependency-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-resources-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>io.fabric8</groupId>
-                <artifactId>docker-maven-plugin</artifactId>
-            </plugin>
-        </plugins>
-
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-dependency-plugin</artifactId>
-                    <executions>
-                        <execution>
-                            <id>download-platform-jar</id>
-                            <phase>prepare-package</phase>
-                            <goals>
-                                <goal>copy</goal>
-                            </goals>
-                            <configuration>
-                                <artifactItems>
-                                    <dependency>
-                                        <groupId>${project.groupId}</groupId>
-                                        <artifactId>${service.server}</artifactId>
-                                        <version>${project.version}</version>
-                                        <type>jar</type>
-                                        <overWrite>true</overWrite>
-                                        <destFileName>${platform.jar}</destFileName>
-                                    </dependency>
-                                </artifactItems>
-                                <outputDirectory>${docker.build.directory}</outputDirectory>
-                            </configuration>
-                        </execution>
-                    </executions>
-                </plugin>
-                <plugin>
-                    <groupId>io.fabric8</groupId>
-                    <artifactId>docker-maven-plugin</artifactId>
-                    <configuration>
-                        <images>
-                            <image>
-                                <name>${docker.image.name}</name>
-                                <build>
-                                    <dockerFileDir>${docker.build.directory}</dockerFileDir>
-                                    <args>
-                                        <PLATFORM_JAR>${platform.jar}</PLATFORM_JAR>
-                                    </args>
-                                    <tags>
-                                        <tag>${docker.image.version}</tag>
-                                        <tag>latest</tag>
-                                    </tags>
-                                </build>
-                            </image>
-                        </images>
-                    </configuration>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
-</project>

layoutparser-service-image/src/main/docker/Dockerfile

@@ -1,9 +0,0 @@
-FROM red/base-image:2.0.2
-
-ARG PLATFORM_JAR
-
-ENV PLATFORM_JAR ${PLATFORM_JAR}
-
-ENV USES_ELASTICSEARCH false
-
-COPY ["${PLATFORM_JAR}", "/"]

layoutparser-service/layoutparser-service-internal-api/build.gradle.kts

@@ -0,0 +1,10 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.4"
+}
+
+description = "layoutparser-service-internal-api"
+
+dependencies {
+    implementation("io.swagger.core.v3:swagger-annotations:2.2.15")
+}

layoutparser-service/layoutparser-service-internal-api/pom.xml

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>1.0.0</version>
-    </parent>
-
-    <artifactId>layoutparser-service-internal-api</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <version>1.18.26</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>31.1-jre</version>
-        </dependency>
-    </dependencies>
-
-</project>

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/AtomicPositionBlockData.java

@@ -1,19 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class AtomicPositionBlockData {
-
-    Long id;
-    int[] stringIdxToPositionIdx;
-    float[][] positions;
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/AtomicTextBlockData.java

@@ -1,23 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class AtomicTextBlockData {
-
-    Long id;
-    Long page;
-    String searchText;
-    int numberOnPage;
-    int start;
-    int end;
-    int[] lineBreaks;
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/DocumentData.java

@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class DocumentData {
-
-    PageData[] pages;
-    AtomicTextBlockData[] atomicTextBlocks;
-    AtomicPositionBlockData[] atomicPositionBlocks;
-    TableOfContentsData tableOfContents;
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/PageData.java

@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class PageData {
-
-    int number;
-    int height;
-    int width;
-    int rotation;
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/TableOfContentsData.java

@@ -1,93 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
-
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableOfContentsData {
-
-    EntryData root;
-
-
-    public EntryData get(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            return root;
-        }
-        EntryData entry = root.subEntries.get(tocId.get(0));
-        for (int id : tocId.subList(1, tocId.size())) {
-            entry = entry.subEntries.get(id);
-        }
-        return entry;
-    }
-
-
-    public Stream<EntryData> streamAllEntries() {
-
-        return Stream.concat(Stream.of(root), root.subEntries.stream()).flatMap(TableOfContentsData::flatten);
-    }
-
-
-    public String toString() {
-
-        return String.join("\n", streamAllEntries().map(EntryData::toString).toList());
-    }
-
-
-    private static Stream<EntryData> flatten(EntryData entry) {
-
-        return Stream.concat(Stream.of(entry), entry.subEntries.stream().flatMap(TableOfContentsData::flatten));
-    }
-
-
-    @Builder
-    @Getter
-    @AllArgsConstructor
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    public static class EntryData {
-
-        NodeType type;
-        int[] tocId;
-        Long[] atomicBlocks;
-        Long[] pages;
-        Map<String, String> properties;
-        List<EntryData> subEntries;
-
-
-        @Override
-        public String toString() {
-
-            StringBuilder sb = new StringBuilder();
-            sb.append("[");
-            for (int i : tocId) {
-                sb.append(i);
-                sb.append(",");
-            }
-            sb.delete(sb.length() - 1, sb.length());
-            sb.append("]: ");
-
-            sb.append(type);
-            sb.append(" atbs = ");
-            sb.append(atomicBlocks.length);
-
-            return sb.toString();
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentData.java

@@ -0,0 +1,28 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@Schema(description = "Object containing the complete document layout parsing information. It is split into 4 categories, structure, text, positions and pages: " + "The document tree structure of SemanticNodes such as Section, Paragraph, Headline, etc. " + "The text, which is stored as separate blocks of data. " + "The text positions, which are also stored as separate blocks. The Blocks are equal to the text blocks in length and order. " + "The page information.")
+public class DocumentData implements Serializable {
+
+    @Schema(description = "Contains information about the document's pages.")
+    DocumentPage[] documentPages;
+    @Schema(description = "Contains information about the document's text.")
+    DocumentTextData[] documentTextData;
+    @Schema(description = "Contains information about the document's text positions.")
+    DocumentPositionData[] documentPositions;
+    @Schema(description = "Contains information about the document's semantic structure.")
+    DocumentStructure documentStructure;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPage.java

@@ -0,0 +1,30 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@Schema(description = "Object containing information about the document's pages.")
+public class DocumentPage implements Serializable {
+
+    @Schema(description = "The page number, starting with 1.")
+    int number;
+    @Schema(description = "The page height in PDF user units.", example = "792")
+    int height;
+    @Schema(description = "The page width in PDF user units.", example = "694")
+    int width;
+    @Schema(description = "The page rotation as specified by the PDF.", example = "90", allowableValues = {"0", "90", "180", "270"})
+    int rotation;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPositionData.java

@@ -0,0 +1,28 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@Schema(description = "Object containing text positional information of a specific text block. A document is split into multiple text blocks, which are supposed to be read in order. Every text block can only occur on a single page.")
+public class DocumentPositionData implements Serializable {
+
+    @Schema(description = "Identifier of the text block.")
+    Long id;
+    @Schema(description = "For each string coordinate in the search text of the text block, the array contains an entry relating the string coordinate to the position coordinate. This is required due to the text and position coordinates not being equal.")
+    int[] stringIdxToPositionIdx;
+    @Schema(description = "The bounding box for each glyph as a rectangle. This matrix is of size (n,4), where n is the number of glyphs in the text block. The second dimension specifies the rectangle with the value x, y, width, height, with x, y specifying the lower left corner. In order to access this information, the stringIdxToPositionIdx array must be used to transform the coordinates.")
+    float[][] positions;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -0,0 +1,144 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.awt.geom.Rectangle2D;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@Schema(description = "Object containing information about the parsed tree structure of the SemanticNodes, such as Section, Paragraph, Headline etc inside of the document.")
+public class DocumentStructure implements Serializable {
+
+    @Schema(description = "The root EntryData represents the Document.")
+    EntryData root;
+
+    @Schema(description = "Object containing the extra field names, a table has in its properties field.")
+    public static class TableProperties implements Serializable {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, an Image has in its properties field.")
+    public static class ImageProperties implements Serializable {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+    }
+
+    @Schema(description = "Object containing the extra field names, a table cell has in its properties field.")
+    public static class TableCellProperties implements Serializable {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
+    public EntryData get(List<Integer> tocId) {
+
+        if (tocId.isEmpty()) {
+            return root;
+        }
+        EntryData entry = root.children.get(tocId.get(0));
+        for (int id : tocId.subList(1, tocId.size())) {
+            entry = entry.children.get(id);
+        }
+        return entry;
+    }
+
+
+    public Stream<EntryData> streamAllEntries() {
+
+        return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentStructure::flatten);
+    }
+
+
+    public String toString() {
+
+        return String.join("\n", streamAllEntries().map(EntryData::toString).toList());
+    }
+
+
+    private static Stream<EntryData> flatten(EntryData entry) {
+
+        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentStructure::flatten));
+    }
+
+
+    @Data
+    @Builder
+    @NoArgsConstructor
+    @AllArgsConstructor
+    @FieldDefaults(level = AccessLevel.PRIVATE)
+    @Schema(description = "Object containing information of a SemanticNode and also structuring the layout with children.")
+    public static class EntryData implements Serializable {
+
+        @Schema(description = "Type of the semantic node.", allowableValues = {"DOCUMENT", "SECTION", "PARAGRAPH", "HEADLINE", "TABLE", "TABLE_CELL", "HEADER", "FOOTER", "IMAGE"})
+        NodeType type;
+        @Schema(description = "Specifies the position in the parsed tree structure.", example = "[1, 0, 2]")
+        int[] treeId;
+        @Schema(description = "Specifies the text block IDs associated with this semantic node. The value should be joined with the DocumentTextData/DocumentPositionData. Is empty, if no text block is directly associated with this semantic node. Only Paragraph, Headline, Header or Footer is directly associated with a text block.", example = "[1]")
+        Long[] atomicBlockIds;
+        @Schema(description = "Specifies the pages this semantic node appears on. The value should be joined with the PageData.", example = "[1, 2, 3]")
+        Long[] pageNumbers;
+        @Schema(description = "Some semantic nodes have additional information, this information is stored in this Map. The extra fields are specified by the Properties subclasses.", example = "For a Table: {\"numberOfRows\": 3, \"numberOfCols\": 4}")
+        Map<String, String> properties;
+        @Schema(description = "All child Entries of this Entry.", example = "[1, 2, 3]")
+        List<EntryData> children;
+        @Schema(description = "Describes the origin of the semantic node", example = "[ALGORITHM]")
+        Set<LayoutEngine> engines;
+
+
+        @Override
+        public String toString() {
+
+            StringBuilder sb = new StringBuilder();
+            sb.append("[");
+            for (int i : treeId) {
+                sb.append(i);
+                sb.append(",");
+            }
+            sb.delete(sb.length() - 1, sb.length());
+            sb.append("]: ");
+
+            sb.append(type);
+            sb.append(" atbs = ");
+            sb.append(atomicBlockIds.length);
+
+            return sb.toString();
+        }
+
+    }
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentTextData.java

@@ -0,0 +1,36 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.io.Serializable;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@Schema(description = "Object containing text information of a specific text block. A document is split into multiple text blocks, which are supposed to be read in order. Every text block can only occur on a single page.")
+public class DocumentTextData implements Serializable {
+
+    @Schema(description = "Identifier of the text block.")
+    Long id;
+    @Schema(description = "The page the text block occurs on.")
+    Long page;
+    @Schema(description = "The text the text block.")
+    String searchText;
+    @Schema(description = "Each text block is assigned a number on a page, starting from 0.")
+    int numberOnPage;
+    @Schema(description = "The text blocks are ordered, this number represents the start of the text block as a string offset.")
+    int start;
+    @Schema(description = "The text blocks are ordered, this number represents the end of the text block as a string offset.")
+    int end;
+    @Schema(description = "The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.", example = "[5, 10]")
+    int[] lineBreaks;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java

@@ -0,0 +1,6 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+public enum LayoutEngine {
+    ALGORITHM,
+    AI
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeType.java

@@ -0,0 +1,22 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.io.Serializable;
+import java.util.Locale;
+
+public enum NodeType implements Serializable {
+    DOCUMENT,
+    SECTION,
+    HEADLINE,
+    PARAGRAPH,
+    TABLE,
+    TABLE_CELL,
+    IMAGE,
+    HEADER,
+    FOOTER;
+
+
+    public String toString() {
+
+        return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
+    }
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedSectionText.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Schema(description = "Object containing a simplified version, which contains almost exclusively text, of the document structure Section class.")
+public class SimplifiedSectionText {
+
+    @Schema(description = "The number of this Section. This is used to map the simplified section text back to the original Section.")
+    private int sectionNumber;
+    @Schema(description = "The text in this Section.")
+    private String text;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -0,0 +1,24 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Schema(description = "Object containing a simplified version, which contains almost exclusively text, of the document structure.")
+public class SimplifiedText {
+
+    @Schema(description = "Number of pages in the entire document.")
+    private int numberOfPages;
+    @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
+    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/ParagraphData.java

@@ -0,0 +1,30 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+@Schema(description = "Object containing information about a Paragraph/Headline/Header/Footer.")
+public class ParagraphData {
+
+    @Schema(description = "The text of this Semantic Node, without any linebreaks.", example = "This is some text.")
+    private String text;
+    @Schema(description = "A list of text ranges in string offsets. Every character in any of the ranges is bold.", example = "[0, 15]")
+    List<Range> boldTextBoundaries;
+    @Schema(description = "A list of text ranges in string offsets. Every character in any of the ranges is italic.", example = "[0, 15]")
+    List<Range> italicTextBoundaries;
+    @Schema(description = "The line breaks in the text of this semantic node in string offsets. They are exclusive end. At the end of each semantic node there is an implicit linebreak.", example = "[5, 10]")
+    List<Integer> linebreaks;
+    @Schema(description = "The classification of this Paragraph.", allowableValues = "{paragraph, headline, header, footer}")
+    private String classification;
+
+    @Schema(description = "Describes the text orientation of this semantic node. Any semantic node only has a single text orientation.", allowableValues = "{ZERO, QUARTER_CIRCLE, HALF_CIRCLE, THREE_QUARTER_CIRCLE}")
+    private String orientation;
+    @Schema(description = "Describes the text direction in degrees of this semantic node. Any semantic node only has a single text direction.", minimum = "0", maximum = "359")
+    private int textDirection;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/Range.java

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+
+@Schema(description = "Object specifying the start and end offsets of a text range in string offsets.")
+public record Range(int start, int end) {
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/ResearchDocumentData.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+@Builder
+@Data
+@AllArgsConstructor
+@Schema(description = "Object containing a simplified version of the document structure. This simplified form only knows Paragraphs and Tables. The Paragraph Objects might be a Paragraph, Headline, Header or Footer.")
+public class ResearchDocumentData {
+
+    @Schema(description = "File name of the original uploaded file.")
+    String originalFile;
+    @Schema(description = "A List of all paragraphs/headline or table objects, that have been parsed in this document.")
+    List<StructureObject> structureObjects;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/RowData.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+@Schema(description = "Object containing information about a Table Row.")
+public class RowData {
+
+    @Schema(description = "Boolean indicating whether this table row is classified as a header row.")
+    boolean header;
+    @Schema(description = "A list of Objects containing information about the text in each cell of this row.")
+    List<ParagraphData> cellText;
+    @Schema(description = "The bounding box of this StructureObject. Is always exactly 4 values representing x, y, w, h, where x, y specify the lower left corner.")
+    float[] bBox;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/StructureObject.java

@@ -0,0 +1,27 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+@AllArgsConstructor
+@Schema(description = "Object containing information about either a Paragraph/Headline/Header/Footer or a Table.")
+public class StructureObject {
+
+    @Schema(description = "The ID of this StructureObject.")
+    Integer structureObjectNumber;
+    @Schema(description = "This value indicates the start of the string offsets in this Object, with respect to the reading order.")
+    int page;
+    @Schema(description = "This stringOffset indicates the start of the string offsets in this Object, with respect to the reading order of the entire document. It is equal to the previous' StructureObject stringOffset + its length.")
+    int stringOffset;
+    @Schema(description = "The bounding box of this StructureObject. Is always exactly 4 values representing x, y, w, h, where x, y specify the lower left corner.", example = "[100, 100, 50, 50]")
+    float[] boundingBox;
+    @Schema(description = "Object containing information about a Paragraph/Headline/Header/Footer. Either this or table is null.")
+    ParagraphData paragraph;
+    @Schema(description = "Object containing information about a Table. Either this or paragraph is null.")
+    TableData table;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/TableData.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+@Schema(description = "Object containing information about a Table.")
+public class TableData {
+
+    @Schema(description = "A list of Objects containing information about all rows in this table.")
+    List<RowData> rowData;
+    @Schema(description = "Number of columns in this table.")
+    Integer numberOfCols;
+    @Schema(description = "Number of rows in this table.")
+    Integer numberOfRows;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/DocumentGraph.java

@@ -1,101 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SectionNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentGraph implements SemanticNode {
-
-    Set<PageNode> pages;
-    TableOfContents tableOfContents;
-    Integer numberOfPages;
-    TextBlock textBlock;
-
-
-    public TextBlock buildTextBlock() {
-
-        return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
-    }
-
-
-    public List<SectionNode> getMainSections() {
-
-        return streamChildren().filter(node -> node instanceof SectionNode).map(node -> (SectionNode) node).collect(Collectors.toList());
-    }
-
-
-    public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
-
-        return streamAllNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock);
-    }
-
-
-    public Set<EntityNode> getEntities() {
-
-        return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
-    }
-
-
-    @Override
-    public List<Integer> getTocId() {
-
-        return Collections.emptyList();
-    }
-
-
-    @Override
-    public void setTocId(List<Integer> tocId) {
-
-        throw new UnsupportedOperationException("DocumentGraph is always the root of the Table of Contents");
-    }
-
-
-    private Stream<SemanticNode> streamAllNodes() {
-
-        return tableOfContents.streamAllEntriesInOrder().map(TableOfContents.Entry::getNode);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return NodeType.DOCUMENT + ": " + buildTextBlock().buildSummary();
-    }
-
-
-    @Override
-    public Map<PageNode, Rectangle2D> getBBox() {
-
-        Map<PageNode, Rectangle2D> bBox = new HashMap<>();
-        for (PageNode page : pages) {
-            bBox.put(page, new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight()));
-        }
-        return bBox;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/TableOfContents.java

@@ -1,193 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph;
-
-import static java.lang.String.format;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.stream.Stream;
-
-import com.google.common.hash.Hashing;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Data
-public class TableOfContents {
-
-    private final Entry root;
-
-
-    public TableOfContents(DocumentGraph documentGraph) {
-
-        root = Entry.builder().tocId(Collections.emptyList()).type(NodeType.DOCUMENT).children(new LinkedList<>()).node(documentGraph).build();
-    }
-
-
-    public TextBlock buildTextBlock() {
-
-        return streamAllEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-    }
-
-
-    public List<Integer> createNewMainEntryAndReturnId(NodeType nodeType, SemanticNode node) {
-
-        return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node);
-    }
-
-
-    public List<Integer> createNewChildEntryAndReturnId(List<Integer> parentId, NodeType nodeType, SemanticNode node) {
-
-        if (!entryExists(parentId)) {
-            throw new UnsupportedOperationException(format("parentId %s does not exist!", parentId));
-        }
-
-        Entry parent = getEntryById(parentId);
-        List<Integer> newId = new LinkedList<>(parentId);
-        newId.add(parent.children.size());
-        parent.children.add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build());
-
-        return newId;
-    }
-
-
-    private boolean entryExists(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            return root != null;
-        }
-        Entry entry = root.children.get(tocId.get(0));
-        for (int id : tocId.subList(1, tocId.size())) {
-            if (id >= entry.children.size() || 0 > id) {
-                return false;
-            }
-            entry = entry.children.get(id);
-        }
-        return true;
-    }
-
-
-    public Entry getParentEntryById(List<Integer> tocId) {
-
-        return getEntryById(getParentId(tocId));
-    }
-
-
-    public boolean hasParentById(List<Integer> tocId) {
-
-        return entryExists(getParentId(tocId));
-    }
-
-
-    public Stream<SemanticNode> streamChildrenNodes(List<Integer> tocId) {
-
-        return getEntryById(tocId).children.stream().map(Entry::getNode);
-    }
-
-
-    private static List<Integer> getParentId(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            throw new UnsupportedOperationException("Root has no parent!");
-        }
-        if (tocId.size() < 2) {
-            return Collections.emptyList();
-        }
-        return tocId.subList(0, tocId.size() - 1);
-    }
-
-
-    public Entry getEntryById(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            return root;
-        }
-        Entry entry = root.children.get(tocId.get(0));
-        for (int id : tocId.subList(1, tocId.size())) {
-            entry = entry.children.get(id);
-        }
-        return entry;
-    }
-
-
-    public Stream<Entry> streamMainEntries() {
-
-        return root.children.stream();
-    }
-
-
-    public Stream<Entry> streamAllEntriesInOrder() {
-
-        return Stream.of(root).flatMap(TableOfContents::flatten);
-    }
-
-
-    public Stream<Entry> streamAllSubEntriesInOrder(List<Integer> parentId) {
-
-        return getEntryById(parentId).getChildren().stream().flatMap(TableOfContents::flatten);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return String.join("\n", streamAllEntriesInOrder().map(Entry::toString).toList());
-    }
-
-
-    public String toString(List<Integer> id) {
-
-        return String.join("\n", streamAllSubEntriesInOrder(id).map(Entry::toString).toList());
-    }
-
-
-    private static Stream<Entry> flatten(Entry entry) {
-
-        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(TableOfContents::flatten));
-    }
-
-
-    @Builder
-    @Getter
-    @AllArgsConstructor
-    @FieldDefaults(level = AccessLevel.PRIVATE,  makeFinal = true)
-    public static class Entry {
-
-        List<Integer> tocId;
-        NodeType type;
-        SemanticNode node;
-        List<Entry> children;
-
-
-        @Override
-        public String toString() {
-
-            return node.toString();
-        }
-
-
-        @Override
-        public int hashCode() {
-
-            return Hashing.murmur3_32_fixed().hashString(toString(), StandardCharsets.UTF_8).hashCode();
-        }
-
-
-        @Override
-        public boolean equals(Object o) {
-
-            return o instanceof Entry && o.hashCode() == this.hashCode();
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/entity/EntityNode.java

@@ -1,76 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.entity;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-
-public interface EntityNode {
-
-    /**
-     * This represents the text, which is contained within the boundary of the Entity.
-     *
-     * @return String
-     */
-    String getValue();
-
-
-    /**
-     * The Boundary primarily defines the Entity, all other values may be inferred from it.
-     *
-     * @return Boundary, uniquely identifying this Entity
-     */
-    Boundary getBoundary();
-
-
-    /**
-     * The deepest fully containing node represents the node which is the deepest node in the document tree structure,
-     * whose boundary also fully contains the boundary of this entity.
-     *
-     * @return the deepest fully containing node
-     */
-    SemanticNode getDeepestFullyContainingNode();
-
-
-    /**
-     * The intersecting nodes represent all nodes, whose boundary intersects the boundary of this entity.
-     *
-     * @return all intersecting Nodes
-     */
-    List<SemanticNode> getIntersectingNodes();
-
-
-    void setDeepestFullyContainingNode(SemanticNode semanticNode);
-
-
-    void addIntersectingNode(SemanticNode semanticNode);
-
-
-    void setIntersectingNodes(List<SemanticNode> semanticNodes);
-
-
-    /**
-     * @return all pages this entity intersects.
-     */
-    Set<PageNode> getPages();
-
-
-    void setPages(Set<PageNode> pages);
-
-
-    /**
-     * removes all occurrences of this node in the graph and resets all graph specific fields.
-     */
-    default void removeFromGraph() {
-
-        getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        getPages().forEach(page -> page.getEntities().remove(this));
-        setPages(Collections.emptySet());
-        setDeepestFullyContainingNode(null);
-        setIntersectingNodes(Collections.emptyList());
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/entity/EntityPosition.java

@@ -1,45 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.entity;
-
-import java.awt.geom.Rectangle2D;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import com.google.common.hash.Hashing;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-
-import lombok.AccessLevel;
-import lombok.Builder;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class EntityPosition {
-
-    PageNode pageNode;
-    List<Rectangle2D> rectanglePerLine;
-
-
-    public String getId() {
-
-        return String.valueOf(hashCode());
-    }
-
-
-    @Override
-    public int hashCode() {
-
-        StringBuilder sb = new StringBuilder();
-        sb.append(pageNode.getNumber());
-        rectanglePerLine.forEach(r -> sb.append(r.getX()).append(r.getY()).append(r.getWidth()).append(r.getHeight()));
-        return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-
-        return o instanceof EntityPosition && o.hashCode() == this.hashCode();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/FooterNode.java

@@ -1,53 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class FooterNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TextBlock terminalTextBlock;
-
-    @Builder.Default
-    boolean terminal = true;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return terminalTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.FOOTER + ": " + terminalTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/HeaderNode.java

@@ -1,53 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class HeaderNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TextBlock terminalTextBlock;
-
-    @Builder.Default
-    boolean terminal = true;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return terminalTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.HEADER + ": " + terminalTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/HeadlineNode.java

@@ -1,60 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class HeadlineNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TextBlock terminalTextBlock;
-
-    @Builder.Default
-    boolean terminal = true;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return terminalTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.HEADLINE + ": " + terminalTextBlock.buildSummary();
-    }
-
-
-    @Override
-    public SemanticNode getHeadline() {
-
-        return this;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/ImageNode.java

@@ -1,87 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class ImageNode implements SemanticNode {
-
-    List<Integer> tocId;
-
-    ImageType imageType;
-    boolean transparency;
-    Rectangle2D position;
-
-
-    boolean redaction;
-    boolean ignored;
-
-    @Builder.Default
-    String redactionReason = "";
-    @Builder.Default
-    String legalBasis = "";
-    @Builder.Default
-    int matchedRule = -1;
-
-    @EqualsAndHashCode.Exclude
-    PageNode page;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-    }
-
-
-    @Override
-    public Set<PageNode> getPages() {
-
-        return Collections.singleton(page);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
-    }
-
-
-    @Override
-    public Map<PageNode, Rectangle2D> getBBox() {
-
-        Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
-        bBoxPerPage.put(page, position);
-        return bBoxPerPage;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/ImageType.java

@@ -1,9 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-public enum ImageType {
-    LOGO,
-    FORMULA,
-    SIGNATURE,
-    OTHER,
-    OCR
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/NodeType.java

@@ -1,13 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-public enum NodeType {
-    DOCUMENT,
-    SECTION,
-    HEADLINE,
-    PARAGRAPH,
-    TABLE,
-    TABLE_CELL,
-    IMAGE,
-    HEADER,
-    FOOTER
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/PageNode.java

@@ -1,71 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@Setter
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class PageNode {
-
-    Integer number;
-    Integer height;
-    Integer width;
-    Integer rotation;
-
-    @EqualsAndHashCode.Exclude
-    List<SemanticNode> mainBody;
-    @EqualsAndHashCode.Exclude
-    HeaderNode header;
-    @EqualsAndHashCode.Exclude
-    FooterNode footer;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<ImageNode> images = new HashSet<>();
-
-
-    public TextBlock getMainBodyTextBlock() {
-
-        return mainBody.stream().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-    }
-
-
-    @Override
-    public String toString() {
-
-        return String.valueOf(number);
-    }
-
-
-    @Override
-    public int hashCode() {
-
-        return number;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-
-        return o instanceof PageNode && o.hashCode() == this.hashCode();
-    }
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/ParagraphNode.java

@@ -1,51 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class ParagraphNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TextBlock terminalTextBlock;
-
-    @Builder.Default
-    boolean terminal = true;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return terminalTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/SectionNode.java

@@ -1,63 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class SectionNode implements SemanticNode {
-
-    List<Integer> tocId;
-
-    TextBlock textBlock;
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-        }
-        return textBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().buildSummary();
-    }
-
-
-    public HeadlineNode getHeadline() {
-
-        return streamChildren().filter(node -> node instanceof HeadlineNode)
-                .map(node -> (HeadlineNode) node)
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException("ClassificationSection has no Headline!"));
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/SemanticNode.java

@@ -1,275 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.services.RectangleTransformations;
-
-public interface SemanticNode {
-
-    /**
-     * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity.
-     * So, for a ClassificationSection all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity
-     * If the Node is Terminal, the TerminalTextBlock will be returned instead.
-     *
-     * @return ClassificationTextBlock containing all AtomicTextBlocks that are located under this Node.
-     */
-    TextBlock buildTextBlock();
-
-
-    /**
-     * Any Node maintains its own Set of Entities.
-     * This Set contains all Entities whose boundary intersects the boundary of this node.
-     *
-     * @return Set of all Entities associated with this Node
-     */
-    Set<EntityNode> getEntities();
-
-
-    /**
-     * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's ClassificationTextBlock.
-     *
-     * @return Set of PageNodes this node appears on.
-     */
-    default Set<PageNode> getPages() {
-
-        return buildTextBlock().getPages();
-    }
-
-
-    /**
-     * @return the TableOfContents of the ClassificationDocument this node belongs to
-     */
-    TableOfContents getTableOfContents();
-
-
-    /**
-     * The id is a List of Integers uniquely identifying this node in the TableOfContents.
-     *
-     * @return the TableOfContents ID
-     */
-    List<Integer> getTocId();
-
-
-    /**
-     * This should only be used during graph construction.
-     *
-     * @param tocId List of Integers
-     */
-    void setTocId(List<Integer> tocId);
-
-
-    /**
-     * Traverses the Tree up, until it hits a HeadlineNode or hits a SectionNode which will then return the first HeadlineNode from its children.
-     * Throws NotFoundException if no Headline is found this way
-     *
-     * @return First HeadlineNode found
-     */
-    default SemanticNode getHeadline() {
-
-        return getParent().getHeadline();
-    }
-
-
-    /**
-     * @return boolean indicating wether this Node has a Parent in the TableOfContents
-     */
-    default boolean hasParent() {
-
-        return getTableOfContents().hasParentById(getTocId());
-    }
-
-
-    /**
-     * @return The SemanticNode representing the Parent in the TableOfContents
-     * throws NotFoundException, when no parent is present
-     */
-    default SemanticNode getParent() {
-
-        return getTableOfContents().getParentEntryById(getTocId()).getNode();
-    }
-
-
-    /**
-     * Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden.
-     * Currently only Sections, Images, and Tables are not terminal.
-     * A TableCell might be Terminal depending on its area compared to the page.
-     *
-     * @return boolean, indicating if a Node has direct access to a ClassificationTextBlock
-     */
-    default boolean isTerminal() {
-
-        return false;
-    }
-
-
-    /**
-     * Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden.
-     * Currently only Sections and Tables are not terminal.
-     *
-     * @return AtomicTextBlock
-     */
-    default TextBlock getTerminalTextBlock() {
-
-        throw new UnsupportedOperationException("Only terminal Nodes have access to TerminalTextBlocks!");
-    }
-
-
-    default void setTerminalTextBlock(TextBlock textBlock) {
-
-        throw new UnsupportedOperationException();
-    }
-
-
-    /**
-     * Each AtomicTextBlock has an index on its page, this returns the number of the first AtomicTextBlock underneath this node.
-     * If this node does not have any AtomicTexBlocks underneath it, e.g. an empty TableCell. It returns -1.
-     *
-     * @return Integer representing the number on the page
-     */
-    default Integer getNumberOnPage() {
-
-        TextBlock textBlock = buildTextBlock();
-        if (textBlock.getAtomicTextBlocks().size() > 0) {
-            return buildTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage();
-        } else {
-            return -1;
-        }
-    }
-
-
-    /**
-     * @return true, if this node's ClassificationTextBlock is not empty
-     */
-    default boolean hasText() {
-
-        return buildTextBlock().length() > 0;
-    }
-
-
-    /**
-     * @param string A String which the ClassificationTextBlock might contain
-     * @return true, if this node's ClassificationTextBlock contains the string
-     */
-    default boolean containsString(String string) {
-
-        return buildTextBlock().getSearchText().contains(string);
-    }
-
-
-    /**
-     * @param strings A List of Strings which the ClassificationTextBlock might contain
-     * @return true, if this node's ClassificationTextBlock contains any of the strings
-     */
-    default boolean containsAnyString(List<String> strings) {
-
-        return strings.stream().anyMatch(this::containsString);
-    }
-
-
-    /**
-     * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the EntityNode intersects or even contains the EntityNode.
-     * It sets the fields accordingly and recursively calls this function on all its children.
-     *
-     * @param entityNode EntityNode, which is being inserted into the graph
-     */
-    default void addThisToEntityIfIntersects(EntityNode entityNode) {
-
-        TextBlock textBlock = buildTextBlock();
-        if (textBlock.getBoundary().intersects(entityNode.getBoundary())) {
-
-            if (textBlock.containsBoundary(entityNode.getBoundary())) {
-                entityNode.setDeepestFullyContainingNode(this);
-            }
-
-            entityNode.addIntersectingNode(this);
-            streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entityNode));
-        }
-    }
-
-
-    /**
-     * Streams all children located directly underneath this node in the TableOfContents.
-     *
-     * @return Stream of all children
-     */
-    default Stream<SemanticNode> streamChildren() {
-
-        return getTableOfContents().streamChildrenNodes(getTocId());
-    }
-
-
-    /**
-     * recursively streams all SemanticNodes located underneath this node in the TableOfContents in order.
-     *
-     * @return Stream of all SubNodes
-     */
-    default Stream<SemanticNode> streamAllSubNodes() {
-
-        return getTableOfContents().streamAllSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::getNode);
-    }
-
-
-    /**
-     * @return Boundary of this Node's ClassificationTextBlock
-     */
-    default Boundary getBoundary() {
-
-        return buildTextBlock().getBoundary();
-    }
-
-
-    /**
-     * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
-     * If called on the ClassificationDocument, it will return the cropbox of each page
-     *
-     * @return Rectangle2D fully encapsulating this Node for each page.
-     */
-    default Map<PageNode, Rectangle2D> getBBox() {
-
-        Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
-        if (isTerminal()) {
-            return getBBoxFromTerminalTextBlock(bBoxPerPage);
-        }
-
-        return getBBoxFromChildren(bBoxPerPage);
-    }
-
-
-    /**
-     * TODO this does not yet work for sections spanning multiple columns.
-     *
-     * @param bBoxPerPage initial empty BoundingBox
-     * @return The union of the BoundingBoxes of all children
-     */
-    private Map<PageNode, Rectangle2D> getBBoxFromChildren(Map<PageNode, Rectangle2D> bBoxPerPage) {
-
-        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
-            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
-            return map2;
-        }).orElse(bBoxPerPage);
-    }
-
-
-    /**
-     * @param bBoxPerPage initial empty BoundingBox
-     * @return The union of all BoundingBoxes of the ClassificationTextBlock of this node
-     */
-    private Map<PageNode, Rectangle2D> getBBoxFromTerminalTextBlock(Map<PageNode, Rectangle2D> bBoxPerPage) {
-
-        Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
-        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
-        return bBoxPerPage;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/TableCellNode.java

@@ -1,92 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableCellNode implements SemanticNode {
-
-    List<Integer> tocId;
-    int row;
-    int col;
-    boolean header;
-
-    Rectangle2D bBox;
-
-    @Builder.Default
-    boolean terminal = true;
-    TextBlock terminalTextBlock;
-
-    TextBlock textBlock;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public Map<PageNode, Rectangle2D> getBBox() {
-
-        Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
-        getPages().forEach(page -> bBoxPerPage.put(page, bBox));
-        return bBoxPerPage;
-    }
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        if (terminal) {
-            return terminalTextBlock;
-        }
-
-        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-        }
-        return textBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().buildSummary();
-    }
-
-
-    public boolean hasHeader(String headerString) {
-
-        return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().strip().equals(headerString));
-    }
-
-
-    private Stream<TableCellNode> getHeaders() {
-
-        TableNode tableNode = (TableNode) getParent();
-        return tableNode.streamHeadersForCell(row, col);
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/TableNode.java

@@ -1,73 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TableOfContents tableOfContents;
-
-    Integer numberOfRows;
-    Integer numberOfCols;
-
-    TextBlock textBlock;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    public Stream<TableCellNode> streamTableCells() {
-
-        return streamChildren().map(node -> (TableCellNode) node);
-    }
-
-
-    public Stream<TableCellNode> streamHeaders() {
-
-        return streamTableCells().filter(TableCellNode::isHeader);
-    }
-
-
-    public Stream<TableCellNode> streamHeadersForCell(int row, int col) {
-
-        return streamHeaders().filter(cell -> cell.getRow() == row || cell.getCol() == col);
-    }
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-        }
-        return textBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/textblock/AtomicTextBlock.java

@@ -1,131 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock;
-
-import java.awt.geom.Rectangle2D;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityPosition;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.services.RectangleTransformations;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class AtomicTextBlock implements TextBlock {
-
-    Long id;
-    Integer numberOnPage;
-    PageNode page;
-
-    //string coordinates
-    Boundary boundary;
-    String searchText;
-    List<Integer> lineBreaks;
-
-    //position coordinates
-    List<Integer> stringIdxToPositionIdx;
-    List<Rectangle2D> positions;
-
-    @EqualsAndHashCode.Exclude
-    SemanticNode parent;
-
-
-    @Override
-    public int numberOfLines() {
-
-        return lineBreaks.size() + 1;
-    }
-
-
-    public CharSequence getLine(int lineNumber) {
-
-        if (lineNumber >= numberOfLines() || lineNumber < 0) {
-            throw new IndexOutOfBoundsException(String.format("line %d out of range for AtomicTextBlock with %d lines", lineNumber, numberOfLines()));
-        }
-        if (lineNumber == 0) {
-            return subSequence(boundary.start(), lineBreaks.get(0) + boundary.start());
-        } else if (lineNumber == numberOfLines() - 1) {
-            return subSequence(lineBreaks.get(lineBreaks.size() - 1) + boundary.start(), boundary.end());
-        }
-        return subSequence(lineBreaks.get(lineNumber - 1) + boundary.start(), lineBreaks.get(lineNumber) + boundary.start());
-    }
-
-
-    @Override
-    public List<AtomicTextBlock> getAtomicTextBlocks() {
-
-        return List.of(this);
-    }
-
-
-    @Override
-    public int getNextLinebreak(int fromIndex) {
-
-        return lineBreaks.stream()//
-                .filter(linebreak -> linebreak > fromIndex - boundary.start()) //
-                .findFirst() //
-                .orElse(searchText.length()) + boundary.start();
-    }
-
-
-    @Override
-    public int getPreviousLinebreak(int fromIndex) {
-
-        return lineBreaks.stream()//
-                .filter(linebreak -> linebreak <= fromIndex - boundary.start())//
-                .reduce((a, b) -> b)//
-                .orElse(0) + boundary.start();
-    }
-
-
-    @Override
-    public Rectangle2D getPosition(int stringIdx) {
-
-        return positions.get(stringIdxToPositionIdx.get(stringIdx - boundary.start()));
-    }
-
-
-    @Override
-    public List<Rectangle2D> getPositions(Boundary stringBoundary) {
-
-        if (!containsBoundary(stringBoundary)) {
-            throw new IndexOutOfBoundsException(String.format("%s is out of bounds for %s", stringBoundary, this.boundary));
-        }
-
-        if (stringBoundary.end() == this.boundary.end()) {
-            return positions.subList(stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start()), positions.size());
-        }
-
-        return positions.subList(stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start()),
-                stringIdxToPositionIdx.get(stringBoundary.end() - this.boundary.start()));
-    }
-
-
-    public List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary) {
-
-        List<Rectangle2D> positionsPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList())
-                .stream()
-                .map(this::getPositions)
-                .map(RectangleTransformations::rectangleUnion)
-                .toList();
-
-        return List.of(EntityPosition.builder().rectanglePerLine(positionsPerLine).pageNode(page).build());
-    }
-
-
-    @Override
-    public String toString() {
-
-        return searchText;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/mapper/DocumentDataMapper.java

@@ -1,146 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.mapper;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicTextBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.PageData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.TableOfContentsData;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableCellNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class DocumentDataMapper {
-
-    public DocumentData toDocumentData(DocumentGraph documentGraph) {
-
-        List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
-                .distinct()
-                .map(DocumentDataMapper::toAtomicTextBlockData)
-                .toList();
-
-        List<AtomicPositionBlockData> atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
-                .distinct()
-                .map(DocumentDataMapper::toAtomicPositionBlockData)
-                .toList();
-
-        List<PageData> pageData = documentGraph.getPages().stream().map(DocumentDataMapper::toPageData).toList();
-        TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
-        return DocumentData.builder()
-                .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
-                .atomicPositionBlocks(atomicPositionBlockData.toArray(new AtomicPositionBlockData[0]))
-                .pages(pageData.toArray(new PageData[0]))
-                .tableOfContents(tableOfContentsData)
-                .build();
-    }
-
-
-    private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) {
-
-        return new TableOfContentsData(toEntryData(tableOfContents.getRoot()));
-    }
-
-
-    private TableOfContentsData.EntryData toEntryData(TableOfContents.Entry entry) {
-
-        Long[] atomicTextBlocks;
-
-        if (entry.getNode().isTerminal()) {
-            atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getTerminalTextBlock());
-        } else {
-            atomicTextBlocks = new Long[]{};
-        }
-
-        Map<String, String> properties = switch (entry.getType()) {
-            case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.getNode());
-            case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.getNode());
-            case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.getNode());
-            default -> new HashMap<>();
-        };
-
-        return TableOfContentsData.EntryData.builder()
-                .tocId(toPrimitiveIntArray(entry.getTocId()))
-                .subEntries(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
-                .type(entry.getType())
-                .atomicBlocks(atomicTextBlocks)
-                .pages(entry.getNode().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new))
-                .properties(properties)
-                .build();
-    }
-
-
-    private Long[] toAtomicTextBlockIds(TextBlock textBlock) {
-
-        return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);
-    }
-
-
-    private PageData toPageData(PageNode p) {
-
-        return PageData.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
-    }
-
-
-    private AtomicTextBlockData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {
-
-        return AtomicTextBlockData.builder()
-                .id(atomicTextBlock.getId())
-                .page(atomicTextBlock.getPage().getNumber().longValue())
-                .searchText(atomicTextBlock.getSearchText())
-                .numberOnPage(atomicTextBlock.getNumberOnPage())
-                .start(atomicTextBlock.getBoundary().start())
-                .end(atomicTextBlock.getBoundary().end())
-                .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
-                .build();
-    }
-
-
-    private AtomicPositionBlockData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {
-
-        return AtomicPositionBlockData.builder()
-                .id(atomicTextBlock.getId())
-                .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
-                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
-                .build();
-    }
-
-
-    private float[][] toPrimitiveFloatMatrix(List<Rectangle2D> positions) {
-
-        float[][] positionMatrix = new float[positions.size()][];
-        for (int i = 0; i < positions.size(); i++) {
-            float[] singlePositions = new float[4];
-            singlePositions[0] = (float) positions.get(i).getMinX();
-            singlePositions[1] = (float) positions.get(i).getMinY();
-            singlePositions[2] = (float) positions.get(i).getWidth();
-            singlePositions[3] = (float) positions.get(i).getHeight();
-            positionMatrix[i] = singlePositions;
-        }
-        return positionMatrix;
-    }
-
-
-    private int[] toPrimitiveIntArray(List<Integer> list) {
-
-        int[] array = new int[list.size()];
-        for (int i = 0; i < list.size(); i++) {
-            array[i] = list.get(i);
-        }
-        return array;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/mapper/DocumentGraphMapper.java

@@ -1,229 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.mapper;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType.FOOTER;
-import static com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType.HEADER;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-
-import com.google.common.primitives.Ints;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicTextBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.PageData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.TableOfContentsData;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.FooterNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.HeaderNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.HeadlineNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ParagraphNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SectionNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableCellNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class DocumentGraphMapper {
-
-    public DocumentGraph toDocumentGraph(DocumentData documentData) {
-
-
-        DocumentGraph documentGraph = new DocumentGraph();
-        Context context = new Context(documentData,
-                new TableOfContents(documentGraph),
-                new LinkedList<>(),
-                new LinkedList<>(),
-                Arrays.stream(documentData.getAtomicTextBlocks()).toList(),
-                Arrays.stream(documentData.getAtomicPositionBlocks()).toList());
-
-        context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList());
-
-        context.tableOfContents.getRoot().getChildren().addAll(buildEntries(documentData.getTableOfContents().getRoot().getSubEntries(), context));
-
-        documentGraph.setTableOfContents(context.tableOfContents);
-        documentGraph.setPages(new HashSet<>(context.pages));
-        documentGraph.setNumberOfPages(documentData.getPages().length);
-
-        documentGraph.setTextBlock(documentGraph.buildTextBlock());
-        return documentGraph;
-    }
-
-
-    private List<TableOfContents.Entry> buildEntries(List<TableOfContentsData.EntryData> entries,
-                                                     Context context) {
-
-        List<TableOfContents.Entry> newEntries = new LinkedList<>();
-        for (TableOfContentsData.EntryData entryData : entries) {
-
-            boolean terminal = isTerminal(entryData);
-            List<PageNode> pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList();
-
-            SemanticNode node = switch (entryData.getType()) {
-                case SECTION -> buildSection(context);
-                case PARAGRAPH -> buildParagraph(context, terminal);
-                case HEADLINE -> buildHeadline(context, terminal);
-                case HEADER -> buildHeader(context, terminal);
-                case FOOTER -> buildFooter(context, terminal);
-                case TABLE -> buildTable(context, entryData.getProperties());
-                case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal);
-                case IMAGE -> buildImage(context, entryData.getProperties());
-                default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
-            };
-
-            if (node.isTerminal()) {
-                TextBlock textBlock = toTextBlock(entryData.getAtomicBlocks(), context, node);
-                node.setTerminalTextBlock(textBlock);
-            }
-            List<Integer> tocId = Arrays.stream(entryData.getTocId()).boxed().toList();
-            node.setTocId(tocId);
-
-            if (entryData.getType() == HEADER) {
-                pages.forEach(page -> page.setHeader((HeaderNode) node));
-            } else if (entryData.getType() == FOOTER) {
-                pages.forEach(page -> page.setFooter((FooterNode) node));
-            } else {
-                pages.forEach(page -> page.getMainBody().add(node));
-            }
-            newEntries.add(TableOfContents.Entry.builder().tocId(tocId).type(entryData.getType()).children(buildEntries(entryData.getSubEntries(), context)).node(node).build());
-        }
-        return newEntries;
-    }
-
-
-    private HeadlineNode buildHeadline(Context context, boolean terminal) {
-
-        return HeadlineNode.builder().terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private static boolean isTerminal(TableOfContentsData.EntryData entryData) {
-
-        return entryData.getAtomicBlocks().length > 0;
-    }
-
-
-    private ImageNode buildImage(Context context, Map<String, String> properties) {
-
-        var builder = ImageNode.builder();
-        PropertiesMapper.parseImageProperties(properties, builder);
-        return builder.tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private TableCellNode buildTableCell(Context context, Map<String, String> properties, boolean terminal) {
-
-        TableCellNode.TableCellNodeBuilder builder = TableCellNode.builder();
-        PropertiesMapper.parseTableCellProperties(properties, builder);
-        return builder.terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private TableNode buildTable(Context context, Map<String, String> properties) {
-
-        TableNode.TableNodeBuilder builder = TableNode.builder();
-        PropertiesMapper.parseTableProperties(properties, builder);
-        return TableNode.builder().tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private FooterNode buildFooter(Context context, boolean terminal) {
-
-        return FooterNode.builder().terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private HeaderNode buildHeader(Context context, boolean terminal) {
-
-        return HeaderNode.builder().terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private SectionNode buildSection(Context context) {
-
-        return SectionNode.builder().tableOfContents(context.tableOfContents()).build();
-
-    }
-
-
-    private ParagraphNode buildParagraph(Context context, boolean terminal) {
-
-        return ParagraphNode.builder().terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
-
-        return Arrays.stream(atomicTextBlockIds)
-                .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                        context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                        parent,
-                        context))
-                .collect(new TextBlockCollector());
-    }
-
-
-    private PageNode buildPage(PageData p) {
-
-        return PageNode.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
-    }
-
-
-    private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData,
-                                              AtomicPositionBlockData atomicPositionBlockData,
-                                              SemanticNode parent,
-                                              Context context) {
-
-        return AtomicTextBlock.builder()
-                .id(atomicTextBlockData.getId())
-                .numberOnPage(atomicTextBlockData.getNumberOnPage())
-                .page(getPage(atomicTextBlockData.getPage(), context))
-                .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
-                .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks()))
-                .stringIdxToPositionIdx(Ints.asList(atomicPositionBlockData.getStringIdxToPositionIdx()))
-                .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
-                .parent(parent)
-                .build();
-    }
-
-
-    private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
-
-        return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList();
-    }
-
-
-    private PageNode getPage(Long pageIndex, Context context) {
-
-        return context.pages.stream()
-                .filter(page -> page.getNumber() == Math.toIntExact(pageIndex))
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException(String.format("ClassificationPage with number %d not found", pageIndex)));
-    }
-
-
-    record Context(
-            DocumentData layoutParsingModel,
-            TableOfContents tableOfContents,
-            List<PageNode> pages,
-            List<SectionNode> sections,
-            List<AtomicTextBlockData> atomicTextBlockData,
-            List<AtomicPositionBlockData> atomicPositionBlockData) {
-
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/mapper/PropertiesMapper.java

@@ -1,101 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.mapper;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableCellNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.services.RectangleTransformations;
-
-public class PropertiesMapper {
-
-    public static Map<String, String> buildImageProperties(ImageNode image) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put("imageType", image.getImageType().toString());
-        properties.put("transparency", String.valueOf(image.isTransparency()));
-        properties.put("position", RectangleTransformations.toString(image.getPosition()));
-        return properties;
-    }
-
-
-    public static Map<String, String> buildTableCellProperties(TableCellNode tableCell) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put("row", String.valueOf(tableCell.getRow()));
-        properties.put("col", String.valueOf(tableCell.getCol()));
-        properties.put("header", String.valueOf(tableCell.isHeader()));
-
-        if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
-            throw new IllegalArgumentException("TableCell can only occur on a single page!");
-        }
-        String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
-        properties.put("bBox", bBoxString);
-
-        return properties;
-    }
-
-
-    public static Map<String, String> buildTableProperties(TableNode table) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put("numberOfRows", String.valueOf(table.getNumberOfRows()));
-        properties.put("numberOfCols", String.valueOf(table.getNumberOfCols()));
-        return properties;
-    }
-
-
-    public static void parseImageProperties(Map<String, String> properties, ImageNode.ImageNodeBuilder builder) {
-
-        builder.imageType(parseImageType(properties.get("imageType")));
-        builder.transparency(Boolean.parseBoolean(properties.get("transparency")));
-        builder.position(parseRectangle2D(properties.get("position")));
-    }
-
-
-    public static void parseTableCellProperties(Map<String, String> properties, TableCellNode.TableCellNodeBuilder builder) {
-
-        builder.row(Integer.parseInt(properties.get("row")));
-        builder.col(Integer.parseInt(properties.get("col")));
-        builder.header(Boolean.parseBoolean(properties.get("header")));
-        builder.bBox(parseRectangle2D(properties.get("bBox")));
-    }
-
-
-    public static void parseTableProperties(Map<String, String> properties, TableNode.TableNodeBuilder builder) {
-
-        builder.numberOfRows(Integer.parseInt(properties.get("numberOfRows")));
-        builder.numberOfCols(Integer.parseInt(properties.get("numberOfCols")));
-    }
-
-
-    private static ImageType parseImageType(String imageType) {
-
-        return switch (imageType) {
-            case "LOGO" -> ImageType.LOGO;
-            case "FORMULA" -> ImageType.FORMULA;
-            case "SIGNATURE" -> ImageType.SIGNATURE;
-            case "OCR" -> ImageType.OCR;
-            default -> ImageType.OTHER;
-        };
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return String.format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingFinishedEvent.java

@@ -2,9 +2,19 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 import java.util.Map;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 
 @Builder
-public record LayoutParsingFinishedEvent(Map<String, String> identifier, long duration, int numberOfPages, String message) {
+@Schema(description = "Object containing information about the layout parsing.")
+public record LayoutParsingFinishedEvent(
+        @Schema(description = "General purpose identifier. It is returned exactly the same way it is inserted with the LayoutParsingRequest.")
+        Map<String, String> identifier,//
+        @Schema(description = "The duration of a single layout parsing in ms.")
+        long duration,//
+        @Schema(description = "The number of pages of the parsed document.")
+        int numberOfPages,//
+        @Schema(description = "A general message. It contains some information useful for a developer, like the paths where the files are stored. Not meant to be machine readable.")
+        String message) {
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "LAYOUTPARSING_REQUEST_QUEUE";
-    public static final String LAYOUT_PARSING_DLQ = "LAYOUTPARSING_DLQ";
-    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "LAYOUTPARSING_FINISHED_EVENT_QUEUE";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
+    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_dead_letter_queue";
+    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -3,18 +3,43 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 import java.util.Map;
 import java.util.Optional;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
-
+import lombok.NonNull;
 
 @Builder
+@Schema(description = "Object containing all storage paths the service needs to know.")
 public record LayoutParsingRequest(
+        @Schema(description = "Enum specifying the type of layout parsing to be performed.", allowableValues = "{RedactManager, DocuMine, TAAS}")//
+        @NonNull LayoutParsingType layoutParsingType,
+
+        @Schema(description = "General purpose identifiers. They are not changed by the service at all and are returned as is in the response queue.")//
         Map<String, String> identifier,
-        String originFileStorageId,
-        Optional<String> tablesFileStorageId,
-        Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId) {
+
+        @Schema(description = "Path to the original PDF file.")//
+        @NonNull String originFileStorageId,//
+
+
+        @Schema(description = "Optional Path to the table extraction file.")//
+        Optional<String> tablesFileStorageId,//
+        @Schema(description = "Optional Path to the image classification file.")//
+        Optional<String> imagesFileStorageId,//
+
+        @Schema(description = "Optional Path to the the visual layout parsing service file") Optional<String> visualLayoutParsingFileId,//
+
+        @Schema(description = "Path where the Document Structure File will be stored.")//
+        @NonNull String structureFileStorageId,//
+        @Schema(description = "Path where the Research Data File will be stored.")//
+        String researchDocumentStorageId,//
+        @Schema(description = "Path where the Document Text File will be stored.")//
+        @NonNull String textBlockFileStorageId,//
+        @Schema(description = "Path where the Document Positions File will be stored.")//
+        @NonNull String positionBlockFileStorageId,//
+        @Schema(description = "Path where the Document Pages File will be stored.")//
+        @NonNull String pageFileStorageId,//
+        @Schema(description = "Path where the Simplified Text File will be stored.")//
+        @NonNull String simplifiedTextStorageId,//
+        @Schema(description = "Path where the Viewer Document PDF will be stored.")//
+        @NonNull String viewerDocumentStorageId) {
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java

@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.queue;
+
+public enum LayoutParsingType {
+    REDACT_MANAGER,
+    TAAS,
+    DOCUMINE
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/services/EntityEnrichmentService.java

@@ -1,10 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.services;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-public interface EntityEnrichmentService {
-
-    void enrichEntity(EntityNode entity, TextBlock textBlock);
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/services/EntityInsertionService.java

@@ -1,56 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.services;
-
-import java.util.Collections;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.RequiredArgsConstructor;
-
-@RequiredArgsConstructor
-public class EntityInsertionService {
-
-    private final EntityEnrichmentService entityEnrichmentService;
-
-
-    public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
-
-        try {
-            SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList())
-                    .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
-                    .findFirst()
-                    .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
-
-            containingNode.addThisToEntityIfIntersects(entity);
-
-            TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock();
-            entityEnrichmentService.enrichEntity(entity, textBlock);
-
-            addToPages(entity);
-            addToNodeEntitySets(entity);
-
-        } catch (NoSuchElementException e) {
-            entity.removeFromGraph();
-        }
-    }
-
-
-    private void addToPages(EntityNode entity) {
-
-        Set<PageNode> pages = entity.getDeepestFullyContainingNode().getPages();
-        entity.getPages().addAll(pages);
-        pages.forEach(page -> page.getEntities().add(entity));
-    }
-
-
-    private void addToNodeEntitySets(EntityNode entity) {
-
-        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/services/RectangleTransformations.java

@@ -1,95 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.services;
-
-import static java.lang.String.format;
-
-import java.awt.geom.Area;
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BinaryOperator;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collector;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class RectangleTransformations {
-
-    public static Rectangle2D pad(Rectangle2D rectangle2D, int deltaX, int deltaY) {
-
-        return new Rectangle2D.Double(rectangle2D.getMinX() - deltaX, rectangle2D.getMinY() - deltaY, rectangle2D.getWidth() + 2 * deltaX, rectangle2D.getHeight() + 2 * deltaY);
-    }
-
-
-    public static Rectangle2D bBoxUnionAtomicTextBlock(List<AtomicTextBlock> atomicTextBlocks) {
-
-        return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DUnion());
-    }
-
-
-    public static Rectangle2D rectangleUnion(List<Rectangle2D> rectangle2DList) {
-
-        return rectangle2DList.stream().collect(new Rectangle2DUnion());
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-
-    private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
-
-        @Override
-        public Supplier<Area> supplier() {
-
-            return Area::new;
-        }
-
-
-        @Override
-        public BiConsumer<Area, Rectangle2D> accumulator() {
-
-            return (area, rectangle2D) -> area.add(new Area(rectangle2D));
-        }
-
-
-        @Override
-        public BinaryOperator<Area> combiner() {
-
-            return (area1, area2) -> {
-                area1.add(area2);
-                return area1;
-            };
-        }
-
-
-        @Override
-        public Function<Area, Rectangle2D> finisher() {
-
-            return Area::getBounds2D;
-        }
-
-
-        @Override
-        public Set<Characteristics> characteristics() {
-
-            return Set.of(Characteristics.CONCURRENT, Characteristics.UNORDERED);
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/test/resources/log4j2-test.xml

@@ -0,0 +1,16 @@
+<Configuration>
+
+    <Appenders>
+        <Console name="CONSOLE" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="warn">
+            <AppenderRef ref="CONSOLE"/>
+        </Root>
+        <Logger name="com.iqser" level="info"/>
+    </Loggers>
+
+</Configuration>

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -0,0 +1,27 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.4"
+}
+
+description = "layoutparser-service-processor"
+
+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0"
+
+dependencies {
+    implementation(project(":layoutparser-service-internal-api"))
+    implementation(project(":viewer-doc-processor"))
+
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
+    }
+    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+
+    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
+}

layoutparser-service/layoutparser-service-processor/pom.xml

@@ -1,129 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>1.0.0</version>
-    </parent>
-
-    <artifactId>layoutparser-service-processor</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-internal-api-v1</artifactId>
-            <version>2.36.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>layoutparser-service-internal-api</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>storage-commons</artifactId>
-            <version>1.13.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>spring-commons</artifactId>
-            <version>6.2.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.dslplatform</groupId>
-            <artifactId>dsl-json-java8</artifactId>
-            <version>1.10.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox-tools</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>31.1-jre</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-afterburner</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-jsr310</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-security</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-web</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <optional>true</optional>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-starter-openfeign</artifactId>
-            <version>4.0.2</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-amqp</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
-    </dependencies>
-
-    <repositories>
-        <repository>
-            <id>spring-milestones</id>
-            <name>Spring Milestones</name>
-            <url>https://repo.spring.io/milestone</url>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-        </repository>
-        <repository>
-            <id>spring-snapshots</id>
-            <name>Spring Snapshots</name>
-            <url>https://repo.spring.io/snapshot</url>
-            <releases>
-                <enabled>false</enabled>
-            </releases>
-        </repository>
-    </repositories>
-    <pluginRepositories>
-        <pluginRepository>
-            <id>spring-milestones</id>
-            <name>Spring Milestones</name>
-            <url>https://repo.spring.io/milestone</url>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-        </pluginRepository>
-        <pluginRepository>
-            <id>spring-snapshots</id>
-            <name>Spring Snapshots</name>
-            <url>https://repo.spring.io/snapshot</url>
-            <releases>
-                <enabled>false</enabled>
-            </releases>
-        </pluginRepository>
-    </pluginRepositories>
-
-</project>

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -0,0 +1,375 @@
+package com.knecon.fforesight.service.layoutparser.processor;
+
+import static java.lang.String.format;
+
+import java.awt.geom.Rectangle2D;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
+import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@SuppressWarnings("PMD.CloseResource")
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class LayoutParsingPipeline {
+
+    ImageServiceResponseAdapter imageServiceResponseAdapter;
+    CvTableParsingAdapter cvTableParsingAdapter;
+    LayoutParsingStorageService layoutParsingStorageService;
+    SectionsBuilderService sectionsBuilderService;
+    TaasClassificationService taasClassificationService;
+    RedactManagerClassificationService redactManagerClassificationService;
+    DocuMineClassificationService docuMineClassificationService;
+    SimplifiedSectionTextService simplifiedSectionTextService;
+    BodyTextFrameService bodyTextFrameService;
+    RulingCleaningService rulingCleaningService;
+    TableExtractionService tableExtractionService;
+    TaasBlockificationService taasBlockificationService;
+    DocuMineBlockificationService docuMineBlockificationService;
+    RedactManagerBlockificationService redactManagerBlockificationService;
+    LayoutGridService layoutGridService;
+    ObservationRegistry observationRegistry;
+    VisualLayoutParsingAdapter visualLayoutParsingAdapter;
+
+
+    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
+
+        long start = System.currentTimeMillis();
+        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());
+
+        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
+        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId())
+                .orElse(originFile);
+
+        VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse();
+        if (layoutParsingRequest.visualLayoutParsingFileId()
+                .isPresent()) {
+            visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId()
+                                                                                                         .get());
+        }
+
+        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+        if (layoutParsingRequest.imagesFileStorageId()
+                .isPresent()) {
+            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId()
+                                                                                     .get());
+        }
+
+        TableServiceResponse tableServiceResponse = new TableServiceResponse();
+        if (layoutParsingRequest.tablesFileStorageId()
+                .isPresent()) {
+            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId()
+                                                                                     .get());
+        }
+
+        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+                                                                    originFile,
+                                                                    imageServiceResponse,
+                                                                    tableServiceResponse,
+                                                                    visualLayoutParsingResponse,
+                                                                    layoutParsingRequest.identifier().toString());
+
+        log.info("Building document graph for {}", layoutParsingRequest.identifier());
+
+        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
+
+        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
+
+        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false);
+
+        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, true);
+
+        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
+
+        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
+
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+            log.info("Building research document data for {}", layoutParsingRequest.identifier());
+            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+        }
+
+        if (!viewerDocumentFile.equals(originFile)) {
+            viewerDocumentFile.delete();
+        }
+        originFile.delete();
+
+        return LayoutParsingFinishedEvent.builder()
+                .identifier(layoutParsingRequest.identifier())
+                .numberOfPages(documentGraph.getNumberOfPages())
+                .duration(System.currentTimeMillis() - start)
+                .message(format("""
+                                        Layout parsing has finished in %.02f s.
+                                        identifiers: %s
+                                        %s
+                                        Files have been saved with Ids:
+                                        Structure: %s
+                                        Text: %s
+                                        Positions: %s
+                                        PageData: %s
+                                        Simplified Text: %s
+                                        Viewer Doc: %s""",
+                                ((float) (System.currentTimeMillis() - start)) / 1000,
+                                layoutParsingRequest.identifier(),
+                                buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
+                                layoutParsingRequest.structureFileStorageId(),
+                                layoutParsingRequest.textBlockFileStorageId(),
+                                layoutParsingRequest.positionBlockFileStorageId(),
+                                layoutParsingRequest.pageFileStorageId(),
+                                layoutParsingRequest.simplifiedTextStorageId(),
+                                layoutParsingRequest.viewerDocumentStorageId()))
+                .build();
+
+    }
+
+
+    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
+
+        AtomicReference<Document> documentReference = new AtomicReference<>();
+
+        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry)
+                .contextualName("build-document-graph")
+                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument)));
+
+        return documentReference.get();
+    }
+
+
+    private String buildSemanticNodeCountMessage(int numberOfPages, Map<NodeType, Long> semanticNodeCounts) {
+
+        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
+                             numberOfPages,
+                             semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                             semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                             semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                             semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                             semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                             semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                             semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
+    }
+
+
+    @SneakyThrows
+    @Observed(name = "LayoutParsingPipeline", contextualName = "parse-layout")
+    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
+                                              File originFile,
+                                              ImageServiceResponse imageServiceResponse,
+                                              TableServiceResponse tableServiceResponse,
+                                              VisualLayoutParsingResponse visualLayoutParsingResponse,
+                                              String identifier) {
+
+        PDDocument originDocument = openDocument(originFile);
+        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
+        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
+        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
+        Map<Integer, List<ClassifiedImage>> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse);
+        ClassificationDocument classificationDocument = new ClassificationDocument();
+        List<ClassificationPage> classificationPages = new ArrayList<>();
+
+        long pageCount = originDocument.getNumberOfPages();
+
+        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+
+            if (pageNumber % 100 == 0) {
+                // re-open document every once in a while to save on RAM. This has no significant performance impact.
+                // This is due to PDFBox caching all images and some other stuff with Soft References. This dereferences them and forces the freeing of memory.
+                originDocument.close();
+                originDocument = openDocument(originFile);
+            }
+
+            if (pageNumber % 100 == 0 || pageNumber == pageCount || pageNumber == 1) {
+                log.info("Extracting text on Page {} for {}", pageNumber, identifier);
+            }
+
+            classificationDocument.setPages(classificationPages);
+            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+            PDPage pdPage = originDocument.getPage(pageNumber - 1);
+            stripper.setPageNumber(pageNumber);
+            stripper.setStartPage(pageNumber);
+            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE)) {
+                stripper.setSortByPosition(true);
+            }
+            stripper.getText(originDocument);
+
+            PDRectangle pdr = pdPage.getMediaBox();
+
+            int rotation = pdPage.getRotation();
+            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+            PDRectangle cropbox = pdPage.getCropBox();
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
+
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            };
+            classificationPage.setCleanRulings(cleanRulings);
+            classificationPage.setRotation(rotation);
+            classificationPage.setLandscape(isLandscape);
+            classificationPage.setPageNumber(pageNumber);
+            classificationPage.setPageWidth(cropbox.getWidth());
+            classificationPage.setPageHeight(cropbox.getHeight());
+
+            // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
+            classificationPage.setMarkedContentBboxPerType(convertMarkedContents(stripper.getMarkedContents()));
+
+            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+                classificationPage.setImages(pdfImages.get(pageNumber));
+                imageServiceResponseAdapter.findOcr(classificationPage);
+            }
+
+            if (signatures.containsKey(pageNumber)) {
+                if (classificationPage.getImages() == null || classificationPage.getImages().size() == 0) {
+                    classificationPage.setImages(signatures.get(pageNumber));
+                } else {
+                    classificationPage.getImages().addAll(signatures.get(pageNumber));
+                }
+            }
+
+            tableExtractionService.extractTables(cleanRulings, classificationPage);
+
+            buildPageStatistics(classificationPage);
+            increaseDocumentStatistics(classificationPage, classificationDocument);
+
+            classificationPages.add(classificationPage);
+        }
+
+        originDocument.close();
+
+        log.info("Calculating BodyTextFrame for {}", identifier);
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+        log.info("Classify TextBlocks for {}", identifier);
+        switch (layoutParsingType) {
+            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
+            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
+        }
+
+        log.info("Building Sections for {}", identifier);
+        sectionsBuilderService.buildSections(classificationDocument);
+        sectionsBuilderService.addImagesToSections(classificationDocument);
+        return classificationDocument;
+    }
+
+
+    private void addNumberOfPagesToTrace(int numberOfPages, long size) {
+
+        if (observationRegistry.getCurrentObservation() != null) {
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
+            observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileSize", String.valueOf(size));
+        }
+    }
+
+
+    @SneakyThrows
+    private PDDocument openDocument(File originFile) {
+
+        PDDocument document = Loader.loadPDF(originFile);
+        document.setAllSecurityToBeRemoved(true);
+        return document;
+    }
+
+
+    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+
+        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
+        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
+        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
+        return markedContentBboxes;
+    }
+
+
+    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
+
+        document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
+        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
+        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
+    }
+
+
+    private void buildPageStatistics(ClassificationPage classificationPage) {
+
+        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
+        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
+                    continue;
+                }
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
+                    classificationPage.getFontCounter().add(word.getFont());
+                    classificationPage.getFontSizeCounter().add(word.getFontSize());
+                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
+                }
+            }
+        }
+
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingService.java

@@ -1,88 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor;
-
-import static java.lang.String.format;
-
-import java.io.IOException;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
-import com.knecon.fforesight.service.layoutparser.internal.api.mapper.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.ClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.PdfParsingService;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.SectionsBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class LayoutParsingService {
-
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final CvTableParsingAdapter cvTableParsingAdapter;
-    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final PdfParsingService pdfParsingService;
-    private final ClassificationService classificationService;
-    private final SectionsBuilderService sectionsBuilderService;
-    private final DocumentGraphFactory documentGraphFactory;
-
-
-    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
-
-        long start = System.currentTimeMillis();
-        PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-
-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
-        }
-
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
-        }
-
-        DocumentGraph documentGraph = parseLayout(originDocument, imageServiceResponse, tableServiceResponse);
-        int numberOfPages = originDocument.getNumberOfPages();
-        originDocument.close();
-
-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-
-        return LayoutParsingFinishedEvent.builder()
-                .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(numberOfPages)
-                .duration(System.currentTimeMillis() - start)
-                .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId()))
-                .build();
-    }
-
-
-    public DocumentGraph parseLayout(PDDocument originDocument, ImageServiceResponse imageServiceResponse, TableServiceResponse tableServiceResponse) {
-
-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
-
-        classificationService.classifyDocument(classificationDocument);
-
-        sectionsBuilderService.buildSections(classificationDocument);
-
-        return documentGraphFactory.buildDocumentGraph(classificationDocument);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingServiceProcessorConfiguration.java

@@ -1,10 +1,24 @@
 package com.knecon.fforesight.service.layoutparser.processor;
 
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+
+import io.micrometer.observation.ObservationRegistry;
+
 @Configuration
 @ComponentScan
 public class LayoutParsingServiceProcessorConfiguration {
 
+
+    @Bean
+    @Autowired
+    public ViewerDocumentService viewerDocumentService(ObservationRegistry registry) {
+
+        return new ViewerDocumentService(registry);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -1,30 +1,31 @@
 package com.knecon.fforesight.service.layoutparser.processor;
 
 import java.io.File;
-import java.io.FileOutputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Optional;
 
-import org.apache.commons.io.IOUtils;
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.io.MemoryUsageSetting;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicTextBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.PageData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.TableOfContentsData;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.multitenancy.TenantContext;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -35,67 +36,70 @@ public class LayoutParsingStorageService {
     private final StorageService storageService;
     private final ObjectMapper objectMapper;
 
+    @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
+    public File getOriginFile(String storageId) throws IOException {
 
-    public PDDocument getOriginFile(String storageId) throws IOException {
+        File tempFile = createTempFile("document", ".pdf");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return tempFile;
+    }
 
-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-            File tempFile = createTempFile("document", ".pdf");
-            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
-                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
-            }
-            return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
+
+    @Observed(name = "LayoutParsingStorageService", contextualName = "get-viewer-doc-file")
+    public Optional<File> getViewerDocFile(String storageId) throws IOException {
+
+        if (!storageService.objectExists(TenantContext.getTenantId(), storageId)) {
+            return Optional.empty();
         }
+        File tempFile = createTempFile("viewerDocument", ".pdf");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        return Optional.of(tempFile);
     }
 
 
     public ImageServiceResponse getImagesFile(String storageId) throws IOException {
 
-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (InputStream inputStream = getObject(storageId)) {
 
-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
         }
     }
 
 
     public TableServiceResponse getTablesFile(String storageId) throws IOException {
 
-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var tableClassificationStream = getObject(storageId)) {
 
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
+        }
+    }
 
+    public VisualLayoutParsingResponse getVisualLayoutParsingFile(String storageId) throws IOException {
+
+        try (InputStream inputStream = getObject(storageId)) {
+            VisualLayoutParsingResponse visualLayoutParsingResponse = objectMapper.readValue(inputStream, VisualLayoutParsingResponse.class);
+            return visualLayoutParsingResponse;
         }
     }
 
 
-    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) throws IOException {
-
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getTableOfContents());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getAtomicTextBlocks());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getAtomicPositionBlocks());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getPages());
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
+    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {
 
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getDocumentTextData());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getDocumentPositions());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getDocumentPages());
     }
 
 
-    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
+    public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {
 
-        PageData[] pageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), PageData[].class);
-        AtomicTextBlockData[] atomicTextBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.textBlockFileStorageId(),
-                AtomicTextBlockData[].class);
-        AtomicPositionBlockData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.positionBlockFileStorageId(),
-                AtomicPositionBlockData[].class);
-        TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.structureFileStorageId(),
-                TableOfContentsData.class);
-
-        return DocumentData.builder()
-                .tableOfContents(tableOfContentsData)
-                .atomicPositionBlocks(atomicPositionBlockData)
-                .atomicTextBlocks(atomicTextBlockData)
-                .pages(pageData)
-                .build();
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.researchDocumentStorageId(), researchDocumentData);
     }
 
 
@@ -123,4 +127,31 @@ public class LayoutParsingStorageService {
         }
     }
 
+
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-simplified-text")
+    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
+    }
+
+
+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
+
+    @SneakyThrows
+    @Observed(name = "LayoutParsingStorageService", contextualName = "store-viewer-document")
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, File out) {
+
+        try (var in = new FileInputStream(out)) {
+            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
+        }
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/CvTableParsingAdapter.java

@@ -1,49 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.CvParsedTableCell;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class CvTableParsingAdapter {
-
-    public Map<Integer, List<com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell>> buildCvParsedTablesPerPage(TableServiceResponse tableServiceResponse) {
-
-        Map<Integer, List<com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell>> tableCells = new HashMap<>();
-        tableServiceResponse.getData()
-                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells())));
-
-        return tableCells;
-    }
-
-
-    private Collection<? extends com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell> convertTableCells(List<CvParsedTableCell> tableCells) {
-
-        List<com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell> cvParsedTableCells = new ArrayList<>();
-
-        tableCells.forEach(t -> cvParsedTableCells.add(com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell.builder()
-                .y0(t.getY0())
-                .x1(t.getX1())
-                .y1(t.getY1())
-                .x0(t.getX0())
-                .width(t.getWidth())
-                .height(t.getHeight())
-                .build()));
-
-        return cvParsedTableCells;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Classification.java

@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Classification {
-
-    private Map<String, Float> probabilities = new HashMap<>();
-    private String label;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/FilterGeometry.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class FilterGeometry {
-
-    private ImageSize imageSize;
-    private Format imageFormat;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Filters.java

@@ -1,15 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Filters {
-
-    private FilterGeometry geometry;
-    private Probability probability;
-    private boolean allPassed;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Format.java

@@ -1,15 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Format {
-
-    private float quotient;
-    private boolean tooTall;
-    private boolean tooWide;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Geometry.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Geometry {
-
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageServiceResponse.java

@@ -1,33 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonAlias;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class ImageServiceResponse {
-
-    private String dossierId;
-    private String fileId;
-
-    @JsonProperty(value = "imageMetadata")
-    @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
-    private List<Metadata> data = new ArrayList<>();
-
-    private List<Metadata> dataCV = new ArrayList<>();
-
-
-    @JsonProperty(value = "imageMetadata")
-    @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
-    public void setData(List<Metadata> data) {this.data = data;}
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageSize.java

@@ -1,15 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class ImageSize {
-
-    private float quotient;
-    private boolean tooLarge;
-    private boolean tooSmall;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Metadata.java

@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Metadata {
-
-    private Classification classification;
-    private Position position;
-    private Geometry geometry;
-    private Filters filters;
-    private boolean alpha;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Position.java

@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Position {
-
-    private float x1;
-    private float x2;
-    private float y1;
-    private float y2;
-    private int pageNumber;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Probability.java

@@ -1,13 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Probability {
-
-    private boolean unconfident;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/CvParsedPageInfo.java

@@ -1,16 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class CvParsedPageInfo {
-
-    private int number;
-    private int rotation;
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/CvParsedTableCell.java

@@ -1,18 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class CvParsedTableCell {
-
-    private float x0;
-    private float y0;
-    private float x1;
-    private float y1;
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/CvParsedTableModel.java

@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class CvParsedTableModel {
-
-    private CvParsedPageInfo pageInfo;
-    private List<CvParsedTableCell> tableCells = new ArrayList<>();
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/AbstractTextContainer.java

@@ -1,71 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
-
-import java.awt.geom.Rectangle2D;
-
-import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextBlockOrientation;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public abstract class AbstractTextContainer {
-
-    protected float minX;
-    protected float maxX;
-    protected float minY;
-    protected float maxY;
-    protected String classification;
-    protected int page;
-
-    private TextBlockOrientation orientation = TextBlockOrientation.NONE;
-
-
-    public abstract String getText();
-
-
-    public boolean containsBlock(ClassificationTextBlock other) {
-
-        return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY();
-    }
-
-
-    public boolean contains(AbstractTextContainer other) {
-
-        return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
-    }
-
-
-    public boolean contains(Rectangle2D other) {
-
-        return other.contains(minX, minY, getWidth(), getHeight());
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getHeight() {
-
-        return maxY - minY;
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getWidth() {
-
-        return maxX - minX;
-    }
-
-
-    public boolean intersectsY(AbstractTextContainer atc) {
-
-        return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/ClassificationFooter.java

@@ -1,16 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
-
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-@Data
-@AllArgsConstructor
-public class ClassificationFooter {
-
-    private List<ClassificationTextBlock> textBlocks;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/ClassificationHeader.java

@@ -1,16 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
-
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-@Data
-@AllArgsConstructor
-public class ClassificationHeader {
-
-    private List<ClassificationTextBlock> textBlocks;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/ClassificationSection.java

@@ -1,38 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.Table;
-
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-public class ClassificationSection implements Comparable {
-
-    private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
-    private List<ClassifiedImage> images = new ArrayList<>();
-    private String headline;
-
-
-    public List<Table> getTables() {
-
-        List<Table> tables = new ArrayList<>();
-        pageBlocks.forEach(block -> {
-            if (block instanceof Table) {
-                tables.add((Table) block);
-            }
-        });
-        return tables;
-    }
-
-
-    @Override
-    public int compareTo(Object o) {
-
-        return 0;
-    }
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/table/TableCell.java

@@ -1,38 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto.table;
-
-import java.awt.geom.Point2D;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.Rectangle;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
-@SuppressWarnings("serial")
-@Data
-@EqualsAndHashCode(callSuper = true)
-@NoArgsConstructor
-public class TableCell extends Rectangle {
-
-    private List<ClassificationTextBlock> textBlocks = new ArrayList<>();
-
-    private List<TableCell> headerCells = new ArrayList<>();
-
-    private boolean isHeaderCell;
-
-
-    public TableCell(Point2D topLeft, Point2D bottomRight) {
-
-        super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()));
-    }
-
-
-    public void addTextBlock(ClassificationTextBlock textBlock) {
-
-        textBlocks.add(textBlock);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/text/TextBlockOrientation.java

@@ -1,8 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto.text;
-
-public enum TextBlockOrientation {
-
-    NONE,
-    LEFT,
-    RIGHT
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/parsing/PDFAreaTextStripper.java

@@ -1,82 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.parsing;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.pdfbox.text.PDFTextStripperByArea;
-import org.apache.pdfbox.text.TextPosition;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-import lombok.Getter;
-import lombok.Setter;
-
-public class PDFAreaTextStripper extends PDFTextStripperByArea {
-
-    @Getter
-    private List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-
-    @Setter
-    private int pageNumber;
-
-
-    public PDFAreaTextStripper() throws IOException {
-
-    }
-
-
-    @Override
-    public void writeString(String text, List<TextPosition> textPositions) throws IOException {
-
-        int startIndex = 0;
-        for (int i = 0; i <= textPositions.size() - 1; i++) {
-
-            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) {
-                startIndex++;
-                continue;
-            }
-
-            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i;
-            }
-
-            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i;
-            }
-
-            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i + 1;
-            }
-        }
-
-        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
-            sublist = sublist.subList(0, sublist.size() - 1);
-        }
-        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-            textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-        }
-        super.writeString(text);
-    }
-
-
-    public void clearPositions() {
-
-        textPositionSequences = new ArrayList<>();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/service/BodyTextFrameService.java

@@ -1,160 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.service;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.FloatFrequencyCounter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.AbstractTextContainer;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.Table;
-import com.knecon.fforesight.service.layoutparser.processor.classification.utils.PositionUtils;
-
-@Service
-public class BodyTextFrameService {
-
-    /**
-     * Adjusts and sets the body text frame to a classificationPage.
-     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the classificationPage rotation.
-     * 0 -> LowerLeft
-     * 90 -> UpperLeft
-     * 180 -> UpperRight
-     * 270 -> LowerRight
-     * The aspect ratio of the classificationPage is also regarded.
-     *
-     * @param classificationPage                   The classificationPage
-     * @param bodyTextFrame          frame that contains the main text on portrait pages
-     * @param landscapeBodyTextFrame frame that contains the main text on landscape pages
-     */
-    public void setBodyTextFrameAdjustedToPage(ClassificationPage classificationPage, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
-
-        Rectangle textFrame = classificationPage.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
-
-        if (classificationPage.getPageWidth() > classificationPage.getPageHeight() && classificationPage.getRotation() == 270) {
-            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getY(), classificationPage.getPageHeight() - textFrame.getTopLeft().getX() - textFrame.getWidth()),
-                    textFrame.getHeight(),
-                    textFrame.getWidth(),
-                    0);
-        } else if (classificationPage.getPageWidth() > classificationPage.getPageHeight() && classificationPage.getRotation() != 0) {
-            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getY(), textFrame.getTopLeft().getX()), textFrame.getHeight(), textFrame.getWidth(), classificationPage.getPageNumber());
-        } else if (classificationPage.getRotation() == 180) {
-            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getX(), classificationPage.getPageHeight() - textFrame.getTopLeft().getY() - textFrame.getHeight()),
-                    textFrame.getWidth(),
-                    textFrame.getHeight(),
-                    0);
-        }
-        classificationPage.setBodyTextFrame(textFrame);
-    }
-
-
-    /**
-     * Calculates the frame that contains the main text, text outside the frame will be e.g. headers or footers.
-     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
-     * 0 -> LowerLeft
-     * 90 -> UpperLeft
-     * 180 -> UpperRight
-     * 270 -> LowerRight
-     * The aspect ratio of the page is also regarded.
-     *
-     * @param classificationPages                   List of all classificationPages
-     * @param documentFontSizeCounter Statistics of the document
-     * @param landscape               Calculate for landscape or portrait
-     * @return Rectangle of the text frame
-     */
-    public Rectangle calculateBodyTextFrame(List<ClassificationPage> classificationPages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
-
-        BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();
-
-        for (ClassificationPage classificationPage : classificationPages) {
-
-            if (classificationPage.getTextBlocks().isEmpty() || landscape != classificationPage.isLandscape()) {
-                continue;
-            }
-
-            for (AbstractTextContainer container : classificationPage.getTextBlocks()) {
-
-                if (container instanceof ClassificationTextBlock) {
-                    ClassificationTextBlock textBlock = (ClassificationTextBlock) container;
-                    if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
-                        continue;
-                    }
-
-                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (approxLineCount < 2.9f) {
-                        continue;
-                    }
-
-                    if (documentFontSizeCounter.getMostPopular() != null && textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
-
-                        expandRectangle(textBlock, classificationPage, expansionsRectangle);
-                    }
-                }
-
-                if (container instanceof Table) {
-                    Table table = (Table) container;
-                    for (List<TableCell> row : table.getRows()) {
-                        for (TableCell cell : row) {
-
-                            if (cell == null || cell.getTextBlocks() == null) {
-                                continue;
-                            }
-                            for (ClassificationTextBlock textBlock : cell.getTextBlocks()) {
-                                expandRectangle(textBlock, classificationPage, expansionsRectangle);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        return new Rectangle(new Point(expansionsRectangle.minX, expansionsRectangle.minY),
-                expansionsRectangle.maxX - expansionsRectangle.minX,
-                expansionsRectangle.maxY - expansionsRectangle.minY,
-                0);
-    }
-
-
-    private void expandRectangle(ClassificationTextBlock textBlock, ClassificationPage classificationPage, BodyTextFrameExpansionsRectangle expansionsRectangle) {
-
-        if (classificationPage.getPageWidth() > classificationPage.getPageHeight() && classificationPage.getRotation() != 0) {
-            if (textBlock.getPdfMinY() < expansionsRectangle.minX) {
-                expansionsRectangle.minX = textBlock.getPdfMinY();
-            }
-            if (textBlock.getPdfMaxY() > expansionsRectangle.maxX) {
-                expansionsRectangle.maxX = textBlock.getPdfMaxY();
-            }
-            if (textBlock.getPdfMinX() < expansionsRectangle.minY) {
-                expansionsRectangle.minY = textBlock.getPdfMinX();
-            }
-            if (textBlock.getPdfMaxX() > expansionsRectangle.maxY) {
-                expansionsRectangle.maxY = textBlock.getPdfMaxX();
-            }
-        } else {
-            if (textBlock.getPdfMinX() < expansionsRectangle.minX) {
-                expansionsRectangle.minX = textBlock.getPdfMinX();
-            }
-            if (textBlock.getPdfMaxX() > expansionsRectangle.maxX) {
-                expansionsRectangle.maxX = textBlock.getPdfMaxX();
-            }
-            if (textBlock.getPdfMinY() < expansionsRectangle.minY) {
-                expansionsRectangle.minY = textBlock.getPdfMinY();
-            }
-            if (textBlock.getPdfMaxY() > expansionsRectangle.maxY) {
-                expansionsRectangle.maxY = textBlock.getPdfMaxY();
-            }
-        }
-    }
-
-
-    private class BodyTextFrameExpansionsRectangle {
-
-        float minX = 10000;
-        float maxX = -100;
-        float minY = 10000;
-        float maxY = -100;
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/service/PdfParsingService.java

@@ -1,134 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.service;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.parsing.PDFLinesTextStripper;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.AbstractTextContainer;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell;
-
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class PdfParsingService {
-
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final BlockificationService blockificationService;
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-
-
-    public ClassificationDocument parseDocument(PDDocument originDocument, Map<Integer, List<CvParsedTableCell>> pdfTableCells, Map<Integer, List<ClassifiedImage>> pdfImages) {
-
-        ClassificationDocument document = new ClassificationDocument();
-        List<ClassificationPage> classificationPages = new ArrayList<>();
-
-        originDocument.setAllSecurityToBeRemoved(true);
-        long pageCount = originDocument.getNumberOfPages();
-
-        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-            parsePage(pdfImages, originDocument, pdfTableCells, document, classificationPages, pageNumber);
-        }
-
-        document.setPages(classificationPages);
-
-        return document;
-    }
-
-
-    @SneakyThrows
-    private void parsePage(Map<Integer, List<ClassifiedImage>> pdfImages,
-                           PDDocument pdDocument,
-                           Map<Integer, List<CvParsedTableCell>> pdfTableCells,
-                           ClassificationDocument document,
-                           List<ClassificationPage> classificationPages,
-                           int pageNumber) {
-
-        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-        stripper.setPageNumber(pageNumber);
-        stripper.setStartPage(pageNumber);
-        stripper.setEndPage(pageNumber);
-        stripper.setPdpage(pdPage);
-        stripper.getText(pdDocument);
-
-        PDRectangle pdr = pdPage.getMediaBox();
-
-        int rotation = pdPage.getRotation();
-        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-        PDRectangle cropbox = pdPage.getCropBox();
-        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                stripper.getRulings(),
-                stripper.getMinCharWidth(),
-                stripper.getMaxCharHeight());
-        ClassificationPage classificationPage = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-        classificationPage.setRotation(rotation);
-        classificationPage.setLandscape(isLandscape);
-        classificationPage.setPageNumber(pageNumber);
-        classificationPage.setPageWidth(cropbox.getWidth());
-        classificationPage.setPageHeight(cropbox.getHeight());
-
-        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-            classificationPage.setImages(pdfImages.get(pageNumber));
-            imageServiceResponseAdapter.findOcr(classificationPage);
-        }
-
-        tableExtractionService.removeRedundantTableCells(cleanRulings, classificationPage);
-        buildPageStatistics(classificationPage);
-        increaseDocumentStatistics(classificationPage, document);
-
-        classificationPages.add(classificationPage);
-    }
-
-
-    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
-
-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-        }
-        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
-        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
-        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
-    }
-
-
-    private void buildPageStatistics(ClassificationPage classificationPage) {
-
-        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
-        for (AbstractTextContainer textBlock : classificationPage.getTextBlocks()) {
-            if (textBlock instanceof ClassificationTextBlock) {
-                if (((ClassificationTextBlock) textBlock).getSequences() == null) {
-                    continue;
-                }
-                for (TextPositionSequence word : ((ClassificationTextBlock) textBlock).getSequences()) {
-                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
-                    classificationPage.getFontCounter().add(word.getFont());
-                    classificationPage.getFontSizeCounter().add(word.getFontSize());
-                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
-                }
-            }
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/DocumentGraphFactory.java

@@ -1,388 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import static java.lang.String.format;
-import static java.util.stream.Collectors.groupingBy;
-import static java.util.stream.Collectors.toList;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.FooterNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.HeaderNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.HeadlineNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ParagraphNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SectionNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableCellNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.AbstractTextContainer;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationFooter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationHeader;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.Table;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-
-@Service
-public class DocumentGraphFactory {
-
-    public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
-
-
-    public DocumentGraph buildDocumentGraph(ClassificationDocument document) {
-
-        TextBlockFactory textBlockFactory = new TextBlockFactory();
-        DocumentGraph documentGraph = new DocumentGraph();
-        Context context = new Context(new TableOfContents(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
-
-        document.getPages().stream().map(this::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
-        document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image));
-        addSections(document, context);
-        addHeaderAndFooterToEachPage(document, context);
-
-        documentGraph.setNumberOfPages(context.pages.size());
-        documentGraph.setPages(context.pages.keySet());
-        documentGraph.setTableOfContents(context.tableOfContents);
-        documentGraph.setTextBlock(documentGraph.buildTextBlock());
-        return documentGraph;
-    }
-
-
-    private void addSections(ClassificationDocument document, Context context) {
-
-        document.getSections().forEach(section -> addSection(null, section.getPageBlocks(), section.getImages(), context));
-    }
-
-
-    private void addSection(SemanticNode parentNode, List<AbstractTextContainer> pageBlocks, List<ClassifiedImage> images, Context context) {
-
-        Map<Integer, List<AbstractTextContainer>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractTextContainer::getPage));
-        SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
-
-        context.sections().add(sectionNode);
-        blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber));
-
-        List<Integer> tocId;
-        if (parentNode == null) {
-            tocId = context.tableOfContents.createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
-        } else {
-            tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
-        }
-        sectionNode.setTocId(tocId);
-        Set<AbstractTextContainer> alreadyMerged = new HashSet<>();
-        for (AbstractTextContainer abstractTextContainer : pageBlocks) {
-
-            if (alreadyMerged.contains(abstractTextContainer)) {
-                continue;
-            }
-
-            if (abstractTextContainer instanceof ClassificationTextBlock) {
-                List<ClassificationTextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractTextContainer, pageBlocks);
-                alreadyMerged.addAll(textBlocks);
-                addParagraphOrHeadline(sectionNode, (ClassificationTextBlock) abstractTextContainer, context, textBlocks);
-            }
-            if (abstractTextContainer instanceof Table) {
-                addTable(sectionNode, (Table) abstractTextContainer, context);
-            }
-        }
-        for (ClassifiedImage image : images) {
-
-            addImage(sectionNode, image, context);
-        }
-    }
-
-
-    private static List<ClassificationTextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) {
-
-        return pageBlocks.stream()
-                .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
-                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
-                .filter(abstractTextContainer -> abstractTextContainer instanceof ClassificationTextBlock)
-                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
-                .map(abstractTextContainer -> (ClassificationTextBlock) abstractTextContainer)
-                .toList();
-    }
-
-
-    private void addSectionNodeToPageNode(Context context, SectionNode sectionNode, Integer pageNumber) {
-
-        PageNode page = getPage(pageNumber, context);
-        page.getMainBody().add(sectionNode);
-    }
-
-
-    private void addTable(SemanticNode parentNode, Table table, Context context) {
-
-        PageNode page = getPage(table.getPage(), context);
-        TableNode tableNode = TableNode.builder().tableOfContents(context.tableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build();
-
-        if (!page.getMainBody().contains(parentNode)) {
-            parentNode.getPages().add(page);
-        }
-
-        page.getMainBody().add(tableNode);
-
-        List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE, tableNode);
-        tableNode.setTocId(tocId);
-
-        addTableCells(table.getRows(), tableNode, context, table.getPage());
-    }
-
-
-    private void addTableCells(List<List<TableCell>> rows, SemanticNode parentNode, Context context, int pageNumber) {
-
-        for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
-            for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, parentNode, pageNumber, context);
-            }
-        }
-    }
-
-
-    private void addTableCell(TableCell cell, int rowIndex, int colIndex, SemanticNode parentNode, int pageNumber, Context context) {
-
-        PageNode page = getPage(pageNumber, context);
-        cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(pageNumber));
-
-        TableCellNode tableCellNode = TableCellNode.builder()
-                .tableOfContents(context.tableOfContents())
-                .row(rowIndex)
-                .col(colIndex)
-                .header(cell.isHeaderCell())
-                .bBox(cell.getBounds2D())
-                .build();
-        page.getMainBody().add(tableCellNode);
-
-        TextBlock textBlock;
-
-        List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode);
-        tableCellNode.setTocId(tocId);
-
-        if (cell.getTextBlocks().isEmpty()) {
-            tableCellNode.setTerminalTextBlock(context.textBlockFactory.emptyTextBlock(parentNode, context, page));
-            tableCellNode.setTerminal(true);
-
-        } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.textBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCellNode, context, page);
-            tableCellNode.setTerminalTextBlock(textBlock);
-            tableCellNode.setTerminal(true);
-
-        } else if (firstTextBlockIsHeadline(cell)) {
-            addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractTextContainer) tb).toList(), Collections.emptyList(), context);
-            tableCellNode.setTerminal(false);
-
-        } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
-            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
-            textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page);
-            tableCellNode.setTerminalTextBlock(textBlock);
-            tableCellNode.setTerminal(true);
-
-        } else {
-            cell.getTextBlocks().forEach(tb -> addParagraphOrHeadline(tableCellNode, tb, context));
-            tableCellNode.setTerminal(false);
-        }
-
-    }
-
-
-    private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(TableCell cell, PageNode page) {
-
-        return cell.getArea() < TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
-    }
-
-
-    private static boolean firstTextBlockIsHeadline(TableCell cell) {
-
-        String classification = cell.getTextBlocks().get(0).getClassification();
-        return classification != null && classification.startsWith("H");
-    }
-
-
-    private void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context) {
-
-        addParagraphOrHeadline(parentNode, originalTextBlock, context, Collections.emptyList());
-    }
-
-
-    private void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context, List<ClassificationTextBlock> textBlocksToMerge) {
-
-        PageNode page = getPage(originalTextBlock.getPage(), context);
-
-        SemanticNode node;
-        if (originalTextBlock.getClassification() != null && originalTextBlock.getClassification().startsWith("H")) {
-            node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build();
-        } else {
-            node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build();
-        }
-
-        page.getMainBody().add(node);
-
-        List<ClassificationTextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
-        textBlocks.add(originalTextBlock);
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
-
-        if (node instanceof HeadlineNode headlineNode) {
-            List<Integer> tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.HEADLINE, node);
-            headlineNode.setTerminalTextBlock(textBlock);
-            headlineNode.setTocId(tocId);
-        }
-        if (node instanceof ParagraphNode paragraphNode) {
-            List<Integer> tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.PARAGRAPH, node);
-            paragraphNode.setTerminalTextBlock(textBlock);
-            paragraphNode.setTocId(tocId);
-        }
-    }
-
-
-    private void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
-
-        PageNode page = getPage(image.getPage(), context);
-        ImageNode imageNode = ImageNode.builder()
-                .imageType(image.getImageType())
-                .position(image.getPosition())
-                .transparency(image.isHasTransparency())
-                .page(page)
-                .tableOfContents(context.tableOfContents())
-                .build();
-        page.getMainBody().add(imageNode);
-
-        List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(sectionNode.getTocId(), NodeType.IMAGE, imageNode);
-        imageNode.setTocId(tocId);
-    }
-
-
-    private void addHeaderAndFooterToEachPage(ClassificationDocument document, Context context) {
-
-        Map<Integer, List<ClassificationTextBlock>> headers = document.getHeaders()
-                .stream()
-                .map(ClassificationHeader::getTextBlocks)
-                .flatMap(List::stream)
-                .collect(groupingBy(AbstractTextContainer::getPage, toList()));
-
-        Map<Integer, List<ClassificationTextBlock>> footers = document.getFooters()
-                .stream()
-                .map(ClassificationFooter::getTextBlocks)
-                .flatMap(List::stream)
-                .collect(groupingBy(AbstractTextContainer::getPage, toList()));
-
-        for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) {
-            if (headers.containsKey(pageIndex)) {
-                addHeader(headers.get(pageIndex), context);
-            } else {
-                addEmptyHeader(pageIndex, context);
-            }
-        }
-
-        for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) {
-            if (footers.containsKey(pageIndex)) {
-                addFooter(footers.get(pageIndex), context);
-            } else {
-                addEmptyFooter(pageIndex, context);
-            }
-        }
-    }
-
-
-    private void addFooter(List<ClassificationTextBlock> textBlocks, Context context) {
-
-        PageNode page = getPage(textBlocks.get(0).getPage(), context);
-        FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
-                footer,
-                context,
-                page);
-        List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
-        footer.setTocId(tocId);
-        footer.setTerminalTextBlock(textBlock);
-        page.setFooter(footer);
-    }
-
-
-    public void addHeader(List<ClassificationTextBlock> textBlocks, Context context) {
-
-        PageNode page = getPage(textBlocks.get(0).getPage(), context);
-        HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
-                header,
-                context,
-                0,
-                page);
-        List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
-        header.setTocId(tocId);
-        header.setTerminalTextBlock(textBlock);
-        page.setHeader(header);
-    }
-
-
-    private void addEmptyFooter(int pageIndex, Context context) {
-
-        PageNode page = getPage(pageIndex, context);
-        FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page);
-        List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
-        footer.setTocId(tocId);
-        footer.setTerminalTextBlock(textBlock);
-        page.setFooter(footer);
-    }
-
-
-    private void addEmptyHeader(int pageIndex, Context context) {
-
-        PageNode page = getPage(pageIndex, context);
-        HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
-        List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
-        header.setTocId(tocId);
-        header.setTerminalTextBlock(textBlock);
-        page.setHeader(header);
-    }
-
-
-    private PageNode buildPage(ClassificationPage p) {
-
-        return PageNode.builder()
-                .height((int) p.getPageHeight())
-                .width((int) p.getPageWidth())
-                .number(p.getPageNumber())
-                .rotation(p.getRotation())
-                .mainBody(new LinkedList<>())
-                .build();
-    }
-
-
-    private PageNode getPage(int pageIndex, Context context) {
-
-        return context.pages.keySet()
-                .stream()
-                .filter(page -> page.getNumber() == pageIndex)
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException(format("ClassificationPage with number %d not found", pageIndex)));
-    }
-
-
-    record Context(
-            TableOfContents tableOfContents, Map<PageNode, AtomicInteger> pages, List<SectionNode> sections, List<ClassifiedImage> images, TextBlockFactory textBlockFactory) {
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/RectangleTransformations.java

@@ -1,105 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import static java.lang.String.format;
-
-import java.awt.geom.Area;
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BinaryOperator;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collector;
-
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.AbstractTextContainer;
-
-public class RectangleTransformations {
-
-    public static Rectangle2D pad(Rectangle2D rectangle2D, int deltaX, int deltaY) {
-
-        return new Rectangle2D.Double(rectangle2D.getMinX() - deltaX, rectangle2D.getMinY() - deltaY, rectangle2D.getWidth() + 2 * deltaX, rectangle2D.getHeight() + 2 * deltaY);
-    }
-
-
-    public static Rectangle2D bBoxUnionAbstractTextContainer(List<AbstractTextContainer> abstractTextContainers) {
-
-        return abstractTextContainers.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion());
-    }
-
-    public static Rectangle2D rectangleUnion(List<Rectangle2D> rectangle2DList) {
-
-        return rectangle2DList.stream().collect(new Rectangle2DUnion());
-    }
-
-
-    public static Rectangle2D toRectangle2D(AbstractTextContainer abstractTextContainer) {
-
-        return new Rectangle2D.Float(abstractTextContainer.getMinX(), abstractTextContainer.getMinY(), abstractTextContainer.getWidth(), abstractTextContainer.getHeight());
-    }
-
-
-    public static Rectangle2D toRectangle2D(PDRectangle rectangle) {
-
-        return new Rectangle2D.Double(rectangle.getLowerLeftX(), rectangle.getLowerLeftY(), rectangle.getWidth(), rectangle.getHeight());
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-
-    private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
-
-        @Override
-        public Supplier<Area> supplier() {
-
-            return Area::new;
-        }
-
-
-        @Override
-        public BiConsumer<Area, Rectangle2D> accumulator() {
-
-            return (area, rectangle2D) -> area.add(new Area(rectangle2D));
-        }
-
-
-        @Override
-        public BinaryOperator<Area> combiner() {
-
-            return (area1, area2) -> {
-                area1.add(area2);
-                return area1;
-            };
-        }
-
-
-        @Override
-        public Function<Area, Rectangle2D> finisher() {
-
-            return Area::getBounds2D;
-        }
-
-
-        @Override
-        public Set<Characteristics> characteristics() {
-
-            return Set.of(Characteristics.CONCURRENT, Characteristics.UNORDERED);
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/SearchTextWithTextPositionFactory.java

@@ -1,156 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.awt.geom.AffineTransform;
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-public class SearchTextWithTextPositionFactory {
-
-    public static final int HEIGHT_PADDING = 2;
-
-
-    public static SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
-
-        if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
-            return SearchTextWithTextPositionModel.builder()
-                    .searchText("")
-                    .lineBreaks(Collections.emptyList())
-                    .positions(Collections.emptyList())
-                    .stringCoordsToPositionCoords(Collections.emptyList())
-                    .build();
-        }
-
-        List<Integer> stringIdxToPositionIdx = new LinkedList<>();
-        List<Integer> lineBreaksStringIdx = new LinkedList<>();
-        StringBuilder sb = new StringBuilder();
-
-        int stringIdx = 0;
-        int positionIdx = 0;
-        int lastHyphenIdx = -3;
-
-        RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
-        RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").position(currentTextPosition.getPosition()).build();
-
-        for (TextPositionSequence word : sequences) {
-            for (int i = 0; i < word.getTextPositions().size(); ++i) {
-
-                currentTextPosition = word.getTextPositions().get(i);
-
-                if (isLineBreak(currentTextPosition, previousTextPosition)) {
-
-                    if (stringIdx - lastHyphenIdx < 3) {
-                        sb.delete(lastHyphenIdx, sb.length());
-                        stringIdxToPositionIdx = stringIdxToPositionIdx.subList(0, lastHyphenIdx);
-                        stringIdx = lastHyphenIdx;
-                        lastHyphenIdx = -3;
-                    }
-                    lineBreaksStringIdx.add(stringIdx);
-                }
-                if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) {
-
-                    if (isHyphen(currentTextPosition.getUnicode())) {
-                        lastHyphenIdx = stringIdx;
-                    }
-                    sb.append(currentTextPosition.getUnicode());
-                    stringIdxToPositionIdx.add(positionIdx);
-                    ++stringIdx;
-                }
-
-                previousTextPosition = currentTextPosition;
-
-                ++positionIdx;
-            }
-
-            previousTextPosition = RedTextPosition.builder().unicode(" ").position(previousTextPosition.getPosition()).build();
-            sb.append(previousTextPosition.getUnicode());
-            stringIdxToPositionIdx.add(positionIdx);
-            ++stringIdx;
-        }
-
-        assert sb.length() == stringIdxToPositionIdx.size();
-
-        List<Rectangle2D> positions = sequences.stream()
-                .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence)))
-                .toList();
-
-        return SearchTextWithTextPositionModel.builder()
-                .searchText(sb.toString())
-                .lineBreaks(lineBreaksStringIdx)
-                .stringCoordsToPositionCoords(stringIdxToPositionIdx)
-                .positions(positions)
-                .build();
-    }
-
-
-    private static boolean isLineBreak(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition) {
-
-        return Objects.equals(currentTextPosition.getUnicode(), "\n") || isDeltaYLargerThanTextHeight(currentTextPosition, previousTextPosition);
-    }
-
-
-    private static boolean isDeltaYLargerThanTextHeight(RedTextPosition currentPosition, RedTextPosition previousPosition) {
-
-        if (previousPosition == null) {
-            return false;
-        }
-
-        float deltaY = Math.abs(currentPosition.getYDirAdj() - previousPosition.getYDirAdj());
-        return deltaY >= currentPosition.getHeightDir();
-    }
-
-
-    private static boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) {
-
-        return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " ");
-    }
-
-
-    private static boolean isHyphen(String unicodeCharacter) {
-
-        return Objects.equals(unicodeCharacter, "-") || //
-                Objects.equals(unicodeCharacter, "~") || //
-                Objects.equals(unicodeCharacter, "‐") || //
-                Objects.equals(unicodeCharacter, "‒") || //
-                Objects.equals(unicodeCharacter, "⁻") || //
-                Objects.equals(unicodeCharacter, "−") || //
-                Objects.equals(unicodeCharacter, "﹣") || //
-                Objects.equals(unicodeCharacter, "゠") || //
-                Objects.equals(unicodeCharacter, "⁓") || //
-                Objects.equals(unicodeCharacter, "‑") || //
-                Objects.equals(unicodeCharacter, "\u00AD");
-    }
-
-
-    private static Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
-
-        float textHeight = sequence.getTextHeight() + HEIGHT_PADDING;
-        Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(),
-                textPosition.getYDirAdj() - textHeight,
-                textPosition.getWidthDirAdj(),
-                textHeight + HEIGHT_PADDING);
-
-        AffineTransform transform = new AffineTransform();
-
-        if (sequence.getDir() == TextDirection.ZERO || sequence.getDir() == TextDirection.HALF_CIRCLE) {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageWidth() / 2f, sequence.getPageHeight() / 2f);
-            transform.translate(0f, sequence.getPageHeight());
-        } else if (sequence.getDir() == TextDirection.QUARTER_CIRCLE) {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageWidth() / 2f, sequence.getPageWidth() / 2f);
-            transform.translate(0f, sequence.getPageWidth());
-        } else {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageHeight() / 2f, sequence.getPageHeight() / 2f);
-            transform.translate(0f, sequence.getPageWidth());
-        }
-        transform.scale(1., -1.);
-
-        return transform.createTransformedShape(rectangle2D).getBounds2D();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/SearchTextWithTextPositionModel.java

@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.awt.geom.Rectangle2D;
-import java.util.List;
-
-import lombok.AccessLevel;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Builder
-@Getter
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class SearchTextWithTextPositionModel {
-
-    String searchText;
-    List<Integer> lineBreaks;
-    List<Integer> stringCoordsToPositionCoords;
-    List<Rectangle2D> positions;
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/TextBlockFactory.java

@@ -1,79 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-
-public class TextBlockFactory {
-
-    AtomicInteger stringOffset;
-    AtomicLong textBlockIdx;
-
-
-    public TextBlockFactory() {
-
-        stringOffset = new AtomicInteger();
-        textBlockIdx = new AtomicLong();
-    }
-
-
-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) {
-
-        Integer numberOnPage = context.pages().get(page).getAndIncrement();
-        return buildAtomicTextBlock(sequences, parent, context, numberOnPage, page);
-    }
-
-
-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences,
-                                                SemanticNode parent,
-                                                DocumentGraphFactory.Context context,
-                                                Integer numberOnPage,
-                                                PageNode page) {
-
-        SearchTextWithTextPositionModel searchTextWithTextPositionModel = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
-        int offset = stringOffset.getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
-
-        return AtomicTextBlock.builder()
-                .id(textBlockIdx.getAndIncrement())
-                .parent(parent)
-                .searchText(searchTextWithTextPositionModel.getSearchText())
-                .numberOnPage(numberOnPage)
-                .page(page)
-                .lineBreaks(searchTextWithTextPositionModel.getLineBreaks())
-                .positions(searchTextWithTextPositionModel.getPositions())
-                .stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords())
-                .boundary(new Boundary(offset, offset + searchTextWithTextPositionModel.getSearchText().length()))
-                .build();
-    }
-
-
-    public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) {
-
-        return emptyTextBlock(parent, context.pages().get(page).getAndIncrement(), page);
-    }
-
-
-    public AtomicTextBlock emptyTextBlock(SemanticNode parent, Integer numberOnPage, PageNode page) {
-
-        return AtomicTextBlock.builder()
-                .id(textBlockIdx.getAndIncrement())
-                .boundary(new Boundary(stringOffset.get(), stringOffset.get()))
-                .searchText("")
-                .lineBreaks(Collections.emptyList())
-                .page(page)
-                .numberOnPage(numberOnPage)
-                .stringIdxToPositionIdx(Collections.emptyList())
-                .positions(Collections.emptyList())
-                .parent(parent)
-                .build();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/TextPositionOperations.java

@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.util.Comparator;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-public class TextPositionOperations {
-
-    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<ClassificationTextBlock> textBlocks) {
-
-        return textBlocks.stream()//
-                .flatMap(tb -> tb.getSequences().stream())//
-                .sorted(Comparator.comparingDouble(TextPositionSequence::getMaxYDirAdj)//
-                        .thenComparing(TextPositionSequence::getMaxXDirAdj))//
-                .toList();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/AbstractPageBlock.java

@@ -0,0 +1,101 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public abstract class AbstractPageBlock {
+
+    @JsonIgnore
+    protected float minX;
+    @JsonIgnore
+    protected float maxX;
+    @JsonIgnore
+    protected float minY;
+    @JsonIgnore
+    protected float maxY;
+    @JsonIgnore
+    protected PageBlockType classification;
+    @JsonIgnore
+    protected int page;
+
+    @JsonIgnore
+    private Orientation orientation = Orientation.NONE;
+
+
+    public abstract String getText();
+
+
+    public boolean isHeadline() {
+
+        return this instanceof TextPageBlock && this.getClassification() != null && this.getClassification().isHeadline();
+    }
+
+
+    public boolean containsBlock(TextPageBlock other) {
+
+        return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY();
+    }
+
+
+    public boolean contains(AbstractPageBlock other) {
+
+        return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
+    }
+
+
+    public boolean contains(Rectangle other) {
+
+        return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft()
+                .getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
+    }
+
+
+    @JsonIgnore
+    public float getHeight() {
+
+        return maxY - minY;
+    }
+
+
+    @JsonIgnore
+    public float getWidth() {
+
+        return maxX - minX;
+    }
+
+
+    public boolean intersectsY(AbstractPageBlock apb) {
+
+        return this.minY <= apb.getMaxY() && this.maxY >= apb.getMinY();
+    }
+
+
+    public boolean almostIntersects(AbstractPageBlock apb, float yThreshold, float xThreshold) {
+
+        return this.almostIntersectsX(apb, xThreshold) && this.almostIntersectsY(apb, yThreshold);
+    }
+
+
+    private boolean almostIntersectsY(AbstractPageBlock apb, float threshold) {
+
+        return this.minY - threshold <= apb.getMaxY() && this.maxY + threshold >= apb.getMinY();
+    }
+
+
+    private boolean almostIntersectsX(AbstractPageBlock apb, float threshold) {
+
+        return this.minX - threshold <= apb.getMaxX() && this.maxX + threshold >= apb.getMinX();
+    }
+
+
+    public abstract boolean isEmpty();
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationDocument.java

@@ -1,10 +1,10 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.ArrayList;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.StringFrequencyCounter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.UnclassifiedText;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText;
 
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -24,4 +24,6 @@ public class ClassificationDocument {
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
     private boolean headlines;
 
+    private long rulesVersion;
+
 }