Merge branch 'RED-7158' into 'main'

RED-7158: add layoutgrid into new ViewerDocument as optional content

See merge request fforesight/layout-parser!34

.gitignore

@@ -18,6 +18,7 @@ target/
 .settings
 .springBeans
 .sts4-cache
+.gradle
 
 ### IntelliJ IDEA ###
 .idea

.gitlab-ci.yml

@@ -1,6 +1,21 @@
-variables:
-  SONAR_PROJECT_KEY: 'fforesight_layout-parser_AYd5quv2mRkBOCG22hvF'
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'
-    file: 'ci-templates/maven_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG

.mvn/wrapper/maven-wrapper.properties

@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.7/apache-maven-3.8.7-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar

buildSrc/build.gradle.kts

@@ -0,0 +1,7 @@
+plugins {
+    `kotlin-dsl`
+}
+
+repositories {
+    gradlePluginPortal()
+}

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -0,0 +1,75 @@
+plugins {
+    `java-library`
+    `maven-publish`
+    pmd
+    checkstyle
+    jacoco
+}
+
+group = "com.knecon.fforesight"
+
+java.sourceCompatibility = JavaVersion.VERSION_17
+java.targetCompatibility = JavaVersion.VERSION_17
+
+tasks.pmdMain {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
+}
+
+tasks.pmdTest {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/test_pmd.xml")
+}
+
+tasks.named<Test>("test") {
+    useJUnitPlatform()
+    reports {
+        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
+    }
+}
+
+tasks.test {
+    finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
+}
+
+tasks.jacocoTestReport {
+    dependsOn(tasks.test) // tests are required to run before generating the report
+    reports {
+        xml.required.set(true)
+        csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
+    }
+}
+
+allprojects {
+    publishing {
+        publications {
+            create<MavenPublication>(name) {
+                from(components["java"])
+            }
+        }
+        repositories {
+            maven {
+                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+                credentials {
+                    username = providers.gradleProperty("mavenUser").getOrNull();
+                    password = providers.gradleProperty("mavenPassword").getOrNull();
+                }
+            }
+        }
+    }
+}
+
+java {
+    withJavadocJar()
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven {
+        url = uri("https://nexus.knecon.com/repository/gindev/")
+        credentials {
+            username = providers.gradleProperty("mavenUser").getOrNull();
+            password = providers.gradleProperty("mavenPassword").getOrNull();
+        }
+    }
+}

config/checkstyle/checkstyle.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <property
+            name="severity"
+            value="error"/>
+    <module name="TreeWalker">
+        <module name="SuppressWarningsHolder"/>
+        <module name="MissingDeprecated"/>
+        <module name="MissingOverride"/>
+        <module name="AnnotationLocation"/>
+        <module name="JavadocStyle"/>
+        <module name="NonEmptyAtclauseDescription"/>
+        <module name="IllegalImport"/>
+        <module name="RedundantImport"/>
+        <module name="RedundantModifier"/>
+        <module name="EmptyBlock"/>
+        <module name="DefaultComesLast"/>
+        <module name="EmptyStatement"/>
+        <module name="EqualsHashCode"/>
+        <module name="ExplicitInitialization"/>
+        <module name="IllegalInstantiation"/>
+        <module name="ModifiedControlVariable"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="PackageDeclaration"/>
+        <module name="ParameterAssignment"/>
+        <module name="SimplifyBooleanExpression"/>
+        <module name="SimplifyBooleanReturn"/>
+        <module name="StringLiteralEquality"/>
+        <module name="OneStatementPerLine"/>
+        <module name="FinalClass"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="UpperEll"/>
+        <module name="OuterTypeFilename"/>
+    </module>
+    <module name="FileTabCharacter"/>
+    <module name="SuppressWarningsFilter"/>
+</module>

config/pmd/pmd.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon ruleset checks the code for bad stuff
+    </description>
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+    </rule>
+
+</ruleset>

config/pmd/test_pmd.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon test ruleset checks the code for bad stuff
+    </description>
+
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="AvoidFieldNameMatchingTypeName"/>
+        <exclude name="TestClassWithoutTestCases"/>
+    </rule>
+
+</ruleset>

gradle.properties.kts

@@ -0,0 +1 @@
+version = 0.1-SNAPSHOT

layoutparser-service-image/pom.xml

@@ -1,99 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-
-
-    <parent>
-        <groupId>com.iqser.red</groupId>
-        <artifactId>platform-docker-dependency</artifactId>
-        <version>1.2.0</version>
-        <relativePath/>
-    </parent>
-
-    <groupId>com.knecon.fforesight</groupId>
-    <artifactId>layoutparser-service-image</artifactId>
-    <version>1.0.0</version>
-    <packaging>pom</packaging>
-
-    <properties>
-        <service.server>layoutparser-service-server</service.server>
-        <platform.jar>${service.server}.jar</platform.jar>
-        <docker.skip.push>false</docker.skip.push>
-        <docker.image.name>${docker.image.prefix}/${service.server}</docker.image.name>
-    </properties>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-dependency-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-resources-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>io.fabric8</groupId>
-                <artifactId>docker-maven-plugin</artifactId>
-            </plugin>
-        </plugins>
-
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-dependency-plugin</artifactId>
-                    <executions>
-                        <execution>
-                            <id>download-platform-jar</id>
-                            <phase>prepare-package</phase>
-                            <goals>
-                                <goal>copy</goal>
-                            </goals>
-                            <configuration>
-                                <artifactItems>
-                                    <dependency>
-                                        <groupId>${project.groupId}</groupId>
-                                        <artifactId>${service.server}</artifactId>
-                                        <version>${project.version}</version>
-                                        <type>jar</type>
-                                        <overWrite>true</overWrite>
-                                        <destFileName>${platform.jar}</destFileName>
-                                    </dependency>
-                                </artifactItems>
-                                <outputDirectory>${docker.build.directory}</outputDirectory>
-                            </configuration>
-                        </execution>
-                    </executions>
-                </plugin>
-                <plugin>
-                    <groupId>io.fabric8</groupId>
-                    <artifactId>docker-maven-plugin</artifactId>
-                    <configuration>
-                        <images>
-                            <image>
-                                <name>${docker.image.name}</name>
-                                <build>
-                                    <dockerFileDir>${docker.build.directory}</dockerFileDir>
-                                    <args>
-                                        <PLATFORM_JAR>${platform.jar}</PLATFORM_JAR>
-                                    </args>
-                                    <tags>
-                                        <tag>${docker.image.version}</tag>
-                                        <tag>latest</tag>
-                                    </tags>
-                                </build>
-                            </image>
-                        </images>
-                    </configuration>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
-</project>

layoutparser-service-image/src/main/docker/Dockerfile

@@ -1,9 +0,0 @@
-FROM red/base-image:2.0.2
-
-ARG PLATFORM_JAR
-
-ENV PLATFORM_JAR ${PLATFORM_JAR}
-
-ENV USES_ELASTICSEARCH false
-
-COPY ["${PLATFORM_JAR}", "/"]

layoutparser-service/layoutparser-service-internal-api/build.gradle.kts

@@ -0,0 +1,6 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.2.2"
+}
+
+description = "layoutparser-service-internal-api"

layoutparser-service/layoutparser-service-internal-api/pom.xml

@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>1.0.0</version>
-    </parent>
-
-    <artifactId>layoutparser-service-internal-api</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <version>1.18.26</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>31.1-jre</version>
-        </dependency>
-    </dependencies>
-
-</project>

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/TableOfContentsData.java

@@ -1,93 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
-
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableOfContentsData {
-
-    EntryData root;
-
-
-    public EntryData get(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            return root;
-        }
-        EntryData entry = root.subEntries.get(tocId.get(0));
-        for (int id : tocId.subList(1, tocId.size())) {
-            entry = entry.subEntries.get(id);
-        }
-        return entry;
-    }
-
-
-    public Stream<EntryData> streamAllEntries() {
-
-        return Stream.concat(Stream.of(root), root.subEntries.stream()).flatMap(TableOfContentsData::flatten);
-    }
-
-
-    public String toString() {
-
-        return String.join("\n", streamAllEntries().map(EntryData::toString).toList());
-    }
-
-
-    private static Stream<EntryData> flatten(EntryData entry) {
-
-        return Stream.concat(Stream.of(entry), entry.subEntries.stream().flatMap(TableOfContentsData::flatten));
-    }
-
-
-    @Builder
-    @Getter
-    @AllArgsConstructor
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    public static class EntryData {
-
-        NodeType type;
-        int[] tocId;
-        Long[] atomicBlocks;
-        Long[] pages;
-        Map<String, String> properties;
-        List<EntryData> subEntries;
-
-
-        @Override
-        public String toString() {
-
-            StringBuilder sb = new StringBuilder();
-            sb.append("[");
-            for (int i : tocId) {
-                sb.append(i);
-                sb.append(",");
-            }
-            sb.delete(sb.length() - 1, sb.length());
-            sb.append("]: ");
-
-            sb.append(type);
-            sb.append(" atbs = ");
-            sb.append(atomicBlocks.length);
-
-            return sb.toString();
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentData.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -12,9 +12,9 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DocumentData {
 
-    PageData[] pages;
-    AtomicTextBlockData[] atomicTextBlocks;
-    AtomicPositionBlockData[] atomicPositionBlocks;
-    TableOfContentsData tableOfContents;
+    DocumentPage[] documentPages;
+    DocumentTextData[] documentTextData;
+    DocumentPositionData[] documentPositions;
+    DocumentStructure documentStructure;
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPage.java

@@ -1,16 +1,18 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class PageData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentPage {
 
     int number;
     int height;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPositionData.java

@@ -1,16 +1,18 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class AtomicPositionBlockData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentPositionData {
 
     Long id;
     int[] stringIdxToPositionIdx;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -0,0 +1,127 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentStructure {
+
+    EntryData root;
+
+
+    public static class TableProperties {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    public static class ImageProperties {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+    }
+
+    public static class TableCellProperties {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
+    public EntryData get(List<Integer> tocId) {
+
+        if (tocId.isEmpty()) {
+            return root;
+        }
+        EntryData entry = root.children.get(tocId.get(0));
+        for (int id : tocId.subList(1, tocId.size())) {
+            entry = entry.children.get(id);
+        }
+        return entry;
+    }
+
+
+    public Stream<EntryData> streamAllEntries() {
+
+        return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentStructure::flatten);
+    }
+
+
+    public String toString() {
+
+        return String.join("\n", streamAllEntries().map(EntryData::toString).toList());
+    }
+
+
+    private static Stream<EntryData> flatten(EntryData entry) {
+
+        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentStructure::flatten));
+    }
+
+
+    @Data
+    @Builder
+    @NoArgsConstructor
+    @AllArgsConstructor
+    @FieldDefaults(level = AccessLevel.PRIVATE)
+    public static class EntryData {
+
+        NodeType type;
+        int[] treeId;
+        Long[] atomicBlockIds;
+        Long[] pageNumbers;
+        Map<String, String> properties;
+        List<EntryData> children;
+
+
+        @Override
+        public String toString() {
+
+            StringBuilder sb = new StringBuilder();
+            sb.append("[");
+            for (int i : treeId) {
+                sb.append(i);
+                sb.append(",");
+            }
+            sb.delete(sb.length() - 1, sb.length());
+            sb.append("]: ");
+
+            sb.append(type);
+            sb.append(" atbs = ");
+            sb.append(atomicBlockIds.length);
+
+            return sb.toString();
+        }
+
+    }
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentTextData.java

@@ -1,16 +1,20 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class AtomicTextBlockData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentTextData {
 
     Long id;
     Long page;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeType.java

@@ -1,4 +1,6 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.util.Locale;
 
 public enum NodeType {
     DOCUMENT,
@@ -9,5 +11,11 @@ public enum NodeType {
     TABLE_CELL,
     IMAGE,
     HEADER,
-    FOOTER
+    FOOTER;
+
+
+    public String toString() {
+
+        return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
+    }
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedSectionText.java

@@ -0,0 +1,17 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedSectionText {
+
+    private int sectionNumber;
+    private String text;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedText {
+
+    private int numberOfPages;
+    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/ParagraphData.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class ParagraphData {
+
+    private String text;
+    List<Range> boldTextBoundaries;
+    List<Range> italicTextBoundaries;
+    List<Integer> linebreaks;
+    private String classification;
+
+    private String orientation;
+    private int textDirection;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/Range.java

@@ -0,0 +1,5 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+public record Range(int start, int end) {
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/ResearchDocumentData.java

@@ -0,0 +1,16 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+@Builder
+@Data
+@AllArgsConstructor
+public class ResearchDocumentData {
+
+    String originalFile;
+    List<StructureObject> structureObjects;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/RowData.java

@@ -0,0 +1,15 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class RowData {
+
+    boolean header;
+    List<ParagraphData> cellText;
+    float[] bBox;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/StructureObject.java

@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+@AllArgsConstructor
+public class StructureObject {
+
+    Integer structureObjectNumber;
+    int page;
+    int stringOffset;
+    float[] boundingBox;
+    ParagraphData paragraph;
+    TableData table;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/TableData.java

@@ -0,0 +1,16 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class TableData {
+
+    List<RowData> rowData;
+    Integer numberOfCols;
+    Integer numberOfRows;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/DocumentGraph.java

@@ -1,101 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SectionNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentGraph implements SemanticNode {
-
-    Set<PageNode> pages;
-    TableOfContents tableOfContents;
-    Integer numberOfPages;
-    TextBlock textBlock;
-
-
-    public TextBlock buildTextBlock() {
-
-        return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
-    }
-
-
-    public List<SectionNode> getMainSections() {
-
-        return streamChildren().filter(node -> node instanceof SectionNode).map(node -> (SectionNode) node).collect(Collectors.toList());
-    }
-
-
-    public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
-
-        return streamAllNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock);
-    }
-
-
-    public Set<EntityNode> getEntities() {
-
-        return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
-    }
-
-
-    @Override
-    public List<Integer> getTocId() {
-
-        return Collections.emptyList();
-    }
-
-
-    @Override
-    public void setTocId(List<Integer> tocId) {
-
-        throw new UnsupportedOperationException("DocumentGraph is always the root of the Table of Contents");
-    }
-
-
-    private Stream<SemanticNode> streamAllNodes() {
-
-        return tableOfContents.streamAllEntriesInOrder().map(TableOfContents.Entry::getNode);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return NodeType.DOCUMENT + ": " + buildTextBlock().buildSummary();
-    }
-
-
-    @Override
-    public Map<PageNode, Rectangle2D> getBBox() {
-
-        Map<PageNode, Rectangle2D> bBox = new HashMap<>();
-        for (PageNode page : pages) {
-            bBox.put(page, new Rectangle2D.Double(0, 0, page.getWidth(), page.getHeight()));
-        }
-        return bBox;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/TableOfContents.java

@@ -1,193 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph;
-
-import static java.lang.String.format;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.stream.Stream;
-
-import com.google.common.hash.Hashing;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Data
-public class TableOfContents {
-
-    private final Entry root;
-
-
-    public TableOfContents(DocumentGraph documentGraph) {
-
-        root = Entry.builder().tocId(Collections.emptyList()).type(NodeType.DOCUMENT).children(new LinkedList<>()).node(documentGraph).build();
-    }
-
-
-    public TextBlock buildTextBlock() {
-
-        return streamAllEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-    }
-
-
-    public List<Integer> createNewMainEntryAndReturnId(NodeType nodeType, SemanticNode node) {
-
-        return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node);
-    }
-
-
-    public List<Integer> createNewChildEntryAndReturnId(List<Integer> parentId, NodeType nodeType, SemanticNode node) {
-
-        if (!entryExists(parentId)) {
-            throw new UnsupportedOperationException(format("parentId %s does not exist!", parentId));
-        }
-
-        Entry parent = getEntryById(parentId);
-        List<Integer> newId = new LinkedList<>(parentId);
-        newId.add(parent.children.size());
-        parent.children.add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build());
-
-        return newId;
-    }
-
-
-    private boolean entryExists(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            return root != null;
-        }
-        Entry entry = root.children.get(tocId.get(0));
-        for (int id : tocId.subList(1, tocId.size())) {
-            if (id >= entry.children.size() || 0 > id) {
-                return false;
-            }
-            entry = entry.children.get(id);
-        }
-        return true;
-    }
-
-
-    public Entry getParentEntryById(List<Integer> tocId) {
-
-        return getEntryById(getParentId(tocId));
-    }
-
-
-    public boolean hasParentById(List<Integer> tocId) {
-
-        return entryExists(getParentId(tocId));
-    }
-
-
-    public Stream<SemanticNode> streamChildrenNodes(List<Integer> tocId) {
-
-        return getEntryById(tocId).children.stream().map(Entry::getNode);
-    }
-
-
-    private static List<Integer> getParentId(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            throw new UnsupportedOperationException("Root has no parent!");
-        }
-        if (tocId.size() < 2) {
-            return Collections.emptyList();
-        }
-        return tocId.subList(0, tocId.size() - 1);
-    }
-
-
-    public Entry getEntryById(List<Integer> tocId) {
-
-        if (tocId.isEmpty()) {
-            return root;
-        }
-        Entry entry = root.children.get(tocId.get(0));
-        for (int id : tocId.subList(1, tocId.size())) {
-            entry = entry.children.get(id);
-        }
-        return entry;
-    }
-
-
-    public Stream<Entry> streamMainEntries() {
-
-        return root.children.stream();
-    }
-
-
-    public Stream<Entry> streamAllEntriesInOrder() {
-
-        return Stream.of(root).flatMap(TableOfContents::flatten);
-    }
-
-
-    public Stream<Entry> streamAllSubEntriesInOrder(List<Integer> parentId) {
-
-        return getEntryById(parentId).getChildren().stream().flatMap(TableOfContents::flatten);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return String.join("\n", streamAllEntriesInOrder().map(Entry::toString).toList());
-    }
-
-
-    public String toString(List<Integer> id) {
-
-        return String.join("\n", streamAllSubEntriesInOrder(id).map(Entry::toString).toList());
-    }
-
-
-    private static Stream<Entry> flatten(Entry entry) {
-
-        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(TableOfContents::flatten));
-    }
-
-
-    @Builder
-    @Getter
-    @AllArgsConstructor
-    @FieldDefaults(level = AccessLevel.PRIVATE,  makeFinal = true)
-    public static class Entry {
-
-        List<Integer> tocId;
-        NodeType type;
-        SemanticNode node;
-        List<Entry> children;
-
-
-        @Override
-        public String toString() {
-
-            return node.toString();
-        }
-
-
-        @Override
-        public int hashCode() {
-
-            return Hashing.murmur3_32_fixed().hashString(toString(), StandardCharsets.UTF_8).hashCode();
-        }
-
-
-        @Override
-        public boolean equals(Object o) {
-
-            return o instanceof Entry && o.hashCode() == this.hashCode();
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/entity/EntityNode.java

@@ -1,76 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.entity;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-
-public interface EntityNode {
-
-    /**
-     * This represents the text, which is contained within the boundary of the Entity.
-     *
-     * @return String
-     */
-    String getValue();
-
-
-    /**
-     * The Boundary primarily defines the Entity, all other values may be inferred from it.
-     *
-     * @return Boundary, uniquely identifying this Entity
-     */
-    Boundary getBoundary();
-
-
-    /**
-     * The deepest fully containing node represents the node which is the deepest node in the document tree structure,
-     * whose boundary also fully contains the boundary of this entity.
-     *
-     * @return the deepest fully containing node
-     */
-    SemanticNode getDeepestFullyContainingNode();
-
-
-    /**
-     * The intersecting nodes represent all nodes, whose boundary intersects the boundary of this entity.
-     *
-     * @return all intersecting Nodes
-     */
-    List<SemanticNode> getIntersectingNodes();
-
-
-    void setDeepestFullyContainingNode(SemanticNode semanticNode);
-
-
-    void addIntersectingNode(SemanticNode semanticNode);
-
-
-    void setIntersectingNodes(List<SemanticNode> semanticNodes);
-
-
-    /**
-     * @return all pages this entity intersects.
-     */
-    Set<PageNode> getPages();
-
-
-    void setPages(Set<PageNode> pages);
-
-
-    /**
-     * removes all occurrences of this node in the graph and resets all graph specific fields.
-     */
-    default void removeFromGraph() {
-
-        getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        getPages().forEach(page -> page.getEntities().remove(this));
-        setPages(Collections.emptySet());
-        setDeepestFullyContainingNode(null);
-        setIntersectingNodes(Collections.emptyList());
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/entity/EntityPosition.java

@@ -1,45 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.entity;
-
-import java.awt.geom.Rectangle2D;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import com.google.common.hash.Hashing;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-
-import lombok.AccessLevel;
-import lombok.Builder;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class EntityPosition {
-
-    PageNode pageNode;
-    List<Rectangle2D> rectanglePerLine;
-
-
-    public String getId() {
-
-        return String.valueOf(hashCode());
-    }
-
-
-    @Override
-    public int hashCode() {
-
-        StringBuilder sb = new StringBuilder();
-        sb.append(pageNode.getNumber());
-        rectanglePerLine.forEach(r -> sb.append(r.getX()).append(r.getY()).append(r.getWidth()).append(r.getHeight()));
-        return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-
-        return o instanceof EntityPosition && o.hashCode() == this.hashCode();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/FooterNode.java

@@ -1,53 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class FooterNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TextBlock terminalTextBlock;
-
-    @Builder.Default
-    boolean terminal = true;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return terminalTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.FOOTER + ": " + terminalTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/HeaderNode.java

@@ -1,53 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class HeaderNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TextBlock terminalTextBlock;
-
-    @Builder.Default
-    boolean terminal = true;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return terminalTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.HEADER + ": " + terminalTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/HeadlineNode.java

@@ -1,60 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class HeadlineNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TextBlock terminalTextBlock;
-
-    @Builder.Default
-    boolean terminal = true;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return terminalTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.HEADLINE + ": " + terminalTextBlock.buildSummary();
-    }
-
-
-    @Override
-    public SemanticNode getHeadline() {
-
-        return this;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/ImageNode.java

@@ -1,87 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class ImageNode implements SemanticNode {
-
-    List<Integer> tocId;
-
-    ImageType imageType;
-    boolean transparency;
-    Rectangle2D position;
-
-
-    boolean redaction;
-    boolean ignored;
-
-    @Builder.Default
-    String redactionReason = "";
-    @Builder.Default
-    String legalBasis = "";
-    @Builder.Default
-    int matchedRule = -1;
-
-    @EqualsAndHashCode.Exclude
-    PageNode page;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-    }
-
-
-    @Override
-    public Set<PageNode> getPages() {
-
-        return Collections.singleton(page);
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
-    }
-
-
-    @Override
-    public Map<PageNode, Rectangle2D> getBBox() {
-
-        Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
-        bBoxPerPage.put(page, position);
-        return bBoxPerPage;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/ImageType.java

@@ -1,9 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-public enum ImageType {
-    LOGO,
-    FORMULA,
-    SIGNATURE,
-    OTHER,
-    OCR
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/PageNode.java

@@ -1,71 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@Setter
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class PageNode {
-
-    Integer number;
-    Integer height;
-    Integer width;
-    Integer rotation;
-
-    @EqualsAndHashCode.Exclude
-    List<SemanticNode> mainBody;
-    @EqualsAndHashCode.Exclude
-    HeaderNode header;
-    @EqualsAndHashCode.Exclude
-    FooterNode footer;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<ImageNode> images = new HashSet<>();
-
-
-    public TextBlock getMainBodyTextBlock() {
-
-        return mainBody.stream().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-    }
-
-
-    @Override
-    public String toString() {
-
-        return String.valueOf(number);
-    }
-
-
-    @Override
-    public int hashCode() {
-
-        return number;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-
-        return o instanceof PageNode && o.hashCode() == this.hashCode();
-    }
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/ParagraphNode.java

@@ -1,51 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class ParagraphNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TextBlock terminalTextBlock;
-
-    @Builder.Default
-    boolean terminal = true;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        return terminalTextBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/SectionNode.java

@@ -1,63 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class SectionNode implements SemanticNode {
-
-    List<Integer> tocId;
-
-    TextBlock textBlock;
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-        }
-        return textBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().buildSummary();
-    }
-
-
-    public HeadlineNode getHeadline() {
-
-        return streamChildren().filter(node -> node instanceof HeadlineNode)
-                .map(node -> (HeadlineNode) node)
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException("ClassificationSection has no Headline!"));
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/SemanticNode.java

@@ -1,275 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.services.RectangleTransformations;
-
-public interface SemanticNode {
-
-    /**
-     * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity.
-     * So, for a ClassificationSection all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity
-     * If the Node is Terminal, the TerminalTextBlock will be returned instead.
-     *
-     * @return ClassificationTextBlock containing all AtomicTextBlocks that are located under this Node.
-     */
-    TextBlock buildTextBlock();
-
-
-    /**
-     * Any Node maintains its own Set of Entities.
-     * This Set contains all Entities whose boundary intersects the boundary of this node.
-     *
-     * @return Set of all Entities associated with this Node
-     */
-    Set<EntityNode> getEntities();
-
-
-    /**
-     * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's ClassificationTextBlock.
-     *
-     * @return Set of PageNodes this node appears on.
-     */
-    default Set<PageNode> getPages() {
-
-        return buildTextBlock().getPages();
-    }
-
-
-    /**
-     * @return the TableOfContents of the ClassificationDocument this node belongs to
-     */
-    TableOfContents getTableOfContents();
-
-
-    /**
-     * The id is a List of Integers uniquely identifying this node in the TableOfContents.
-     *
-     * @return the TableOfContents ID
-     */
-    List<Integer> getTocId();
-
-
-    /**
-     * This should only be used during graph construction.
-     *
-     * @param tocId List of Integers
-     */
-    void setTocId(List<Integer> tocId);
-
-
-    /**
-     * Traverses the Tree up, until it hits a HeadlineNode or hits a SectionNode which will then return the first HeadlineNode from its children.
-     * Throws NotFoundException if no Headline is found this way
-     *
-     * @return First HeadlineNode found
-     */
-    default SemanticNode getHeadline() {
-
-        return getParent().getHeadline();
-    }
-
-
-    /**
-     * @return boolean indicating wether this Node has a Parent in the TableOfContents
-     */
-    default boolean hasParent() {
-
-        return getTableOfContents().hasParentById(getTocId());
-    }
-
-
-    /**
-     * @return The SemanticNode representing the Parent in the TableOfContents
-     * throws NotFoundException, when no parent is present
-     */
-    default SemanticNode getParent() {
-
-        return getTableOfContents().getParentEntryById(getTocId()).getNode();
-    }
-
-
-    /**
-     * Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden.
-     * Currently only Sections, Images, and Tables are not terminal.
-     * A TableCell might be Terminal depending on its area compared to the page.
-     *
-     * @return boolean, indicating if a Node has direct access to a ClassificationTextBlock
-     */
-    default boolean isTerminal() {
-
-        return false;
-    }
-
-
-    /**
-     * Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden.
-     * Currently only Sections and Tables are not terminal.
-     *
-     * @return AtomicTextBlock
-     */
-    default TextBlock getTerminalTextBlock() {
-
-        throw new UnsupportedOperationException("Only terminal Nodes have access to TerminalTextBlocks!");
-    }
-
-
-    default void setTerminalTextBlock(TextBlock textBlock) {
-
-        throw new UnsupportedOperationException();
-    }
-
-
-    /**
-     * Each AtomicTextBlock has an index on its page, this returns the number of the first AtomicTextBlock underneath this node.
-     * If this node does not have any AtomicTexBlocks underneath it, e.g. an empty TableCell. It returns -1.
-     *
-     * @return Integer representing the number on the page
-     */
-    default Integer getNumberOnPage() {
-
-        TextBlock textBlock = buildTextBlock();
-        if (textBlock.getAtomicTextBlocks().size() > 0) {
-            return buildTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage();
-        } else {
-            return -1;
-        }
-    }
-
-
-    /**
-     * @return true, if this node's ClassificationTextBlock is not empty
-     */
-    default boolean hasText() {
-
-        return buildTextBlock().length() > 0;
-    }
-
-
-    /**
-     * @param string A String which the ClassificationTextBlock might contain
-     * @return true, if this node's ClassificationTextBlock contains the string
-     */
-    default boolean containsString(String string) {
-
-        return buildTextBlock().getSearchText().contains(string);
-    }
-
-
-    /**
-     * @param strings A List of Strings which the ClassificationTextBlock might contain
-     * @return true, if this node's ClassificationTextBlock contains any of the strings
-     */
-    default boolean containsAnyString(List<String> strings) {
-
-        return strings.stream().anyMatch(this::containsString);
-    }
-
-
-    /**
-     * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the EntityNode intersects or even contains the EntityNode.
-     * It sets the fields accordingly and recursively calls this function on all its children.
-     *
-     * @param entityNode EntityNode, which is being inserted into the graph
-     */
-    default void addThisToEntityIfIntersects(EntityNode entityNode) {
-
-        TextBlock textBlock = buildTextBlock();
-        if (textBlock.getBoundary().intersects(entityNode.getBoundary())) {
-
-            if (textBlock.containsBoundary(entityNode.getBoundary())) {
-                entityNode.setDeepestFullyContainingNode(this);
-            }
-
-            entityNode.addIntersectingNode(this);
-            streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entityNode));
-        }
-    }
-
-
-    /**
-     * Streams all children located directly underneath this node in the TableOfContents.
-     *
-     * @return Stream of all children
-     */
-    default Stream<SemanticNode> streamChildren() {
-
-        return getTableOfContents().streamChildrenNodes(getTocId());
-    }
-
-
-    /**
-     * recursively streams all SemanticNodes located underneath this node in the TableOfContents in order.
-     *
-     * @return Stream of all SubNodes
-     */
-    default Stream<SemanticNode> streamAllSubNodes() {
-
-        return getTableOfContents().streamAllSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::getNode);
-    }
-
-
-    /**
-     * @return Boundary of this Node's ClassificationTextBlock
-     */
-    default Boundary getBoundary() {
-
-        return buildTextBlock().getBoundary();
-    }
-
-
-    /**
-     * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
-     * If called on the ClassificationDocument, it will return the cropbox of each page
-     *
-     * @return Rectangle2D fully encapsulating this Node for each page.
-     */
-    default Map<PageNode, Rectangle2D> getBBox() {
-
-        Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
-        if (isTerminal()) {
-            return getBBoxFromTerminalTextBlock(bBoxPerPage);
-        }
-
-        return getBBoxFromChildren(bBoxPerPage);
-    }
-
-
-    /**
-     * TODO this does not yet work for sections spanning multiple columns.
-     *
-     * @param bBoxPerPage initial empty BoundingBox
-     * @return The union of the BoundingBoxes of all children
-     */
-    private Map<PageNode, Rectangle2D> getBBoxFromChildren(Map<PageNode, Rectangle2D> bBoxPerPage) {
-
-        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
-            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
-            return map2;
-        }).orElse(bBoxPerPage);
-    }
-
-
-    /**
-     * @param bBoxPerPage initial empty BoundingBox
-     * @return The union of all BoundingBoxes of the ClassificationTextBlock of this node
-     */
-    private Map<PageNode, Rectangle2D> getBBoxFromTerminalTextBlock(Map<PageNode, Rectangle2D> bBoxPerPage) {
-
-        Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
-        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
-        return bBoxPerPage;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/TableCellNode.java

@@ -1,92 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableCellNode implements SemanticNode {
-
-    List<Integer> tocId;
-    int row;
-    int col;
-    boolean header;
-
-    Rectangle2D bBox;
-
-    @Builder.Default
-    boolean terminal = true;
-    TextBlock terminalTextBlock;
-
-    TextBlock textBlock;
-
-    @EqualsAndHashCode.Exclude
-    TableOfContents tableOfContents;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    @Override
-    public Map<PageNode, Rectangle2D> getBBox() {
-
-        Map<PageNode, Rectangle2D> bBoxPerPage = new HashMap<>();
-        getPages().forEach(page -> bBoxPerPage.put(page, bBox));
-        return bBoxPerPage;
-    }
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        if (terminal) {
-            return terminalTextBlock;
-        }
-
-        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-        }
-        return textBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().buildSummary();
-    }
-
-
-    public boolean hasHeader(String headerString) {
-
-        return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().strip().equals(headerString));
-    }
-
-
-    private Stream<TableCellNode> getHeaders() {
-
-        TableNode tableNode = (TableNode) getParent();
-        return tableNode.streamHeadersForCell(row, col);
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/nodes/TableNode.java

@@ -1,73 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Stream;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableNode implements SemanticNode {
-
-    List<Integer> tocId;
-    TableOfContents tableOfContents;
-
-    Integer numberOfRows;
-    Integer numberOfCols;
-
-    TextBlock textBlock;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<EntityNode> entities = new HashSet<>();
-
-
-    public Stream<TableCellNode> streamTableCells() {
-
-        return streamChildren().map(node -> (TableCellNode) node);
-    }
-
-
-    public Stream<TableCellNode> streamHeaders() {
-
-        return streamTableCells().filter(TableCellNode::isHeader);
-    }
-
-
-    public Stream<TableCellNode> streamHeadersForCell(int row, int col) {
-
-        return streamHeaders().filter(cell -> cell.getRow() == row || cell.getCol() == col);
-    }
-
-
-    @Override
-    public TextBlock buildTextBlock() {
-
-        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
-        }
-        return textBlock;
-    }
-
-
-    @Override
-    public String toString() {
-
-        return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().buildSummary();
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/graph/textblock/AtomicTextBlock.java

@@ -1,131 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock;
-
-import java.awt.geom.Rectangle2D;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityPosition;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.services.RectangleTransformations;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class AtomicTextBlock implements TextBlock {
-
-    Long id;
-    Integer numberOnPage;
-    PageNode page;
-
-    //string coordinates
-    Boundary boundary;
-    String searchText;
-    List<Integer> lineBreaks;
-
-    //position coordinates
-    List<Integer> stringIdxToPositionIdx;
-    List<Rectangle2D> positions;
-
-    @EqualsAndHashCode.Exclude
-    SemanticNode parent;
-
-
-    @Override
-    public int numberOfLines() {
-
-        return lineBreaks.size() + 1;
-    }
-
-
-    public CharSequence getLine(int lineNumber) {
-
-        if (lineNumber >= numberOfLines() || lineNumber < 0) {
-            throw new IndexOutOfBoundsException(String.format("line %d out of range for AtomicTextBlock with %d lines", lineNumber, numberOfLines()));
-        }
-        if (lineNumber == 0) {
-            return subSequence(boundary.start(), lineBreaks.get(0) + boundary.start());
-        } else if (lineNumber == numberOfLines() - 1) {
-            return subSequence(lineBreaks.get(lineBreaks.size() - 1) + boundary.start(), boundary.end());
-        }
-        return subSequence(lineBreaks.get(lineNumber - 1) + boundary.start(), lineBreaks.get(lineNumber) + boundary.start());
-    }
-
-
-    @Override
-    public List<AtomicTextBlock> getAtomicTextBlocks() {
-
-        return List.of(this);
-    }
-
-
-    @Override
-    public int getNextLinebreak(int fromIndex) {
-
-        return lineBreaks.stream()//
-                .filter(linebreak -> linebreak > fromIndex - boundary.start()) //
-                .findFirst() //
-                .orElse(searchText.length()) + boundary.start();
-    }
-
-
-    @Override
-    public int getPreviousLinebreak(int fromIndex) {
-
-        return lineBreaks.stream()//
-                .filter(linebreak -> linebreak <= fromIndex - boundary.start())//
-                .reduce((a, b) -> b)//
-                .orElse(0) + boundary.start();
-    }
-
-
-    @Override
-    public Rectangle2D getPosition(int stringIdx) {
-
-        return positions.get(stringIdxToPositionIdx.get(stringIdx - boundary.start()));
-    }
-
-
-    @Override
-    public List<Rectangle2D> getPositions(Boundary stringBoundary) {
-
-        if (!containsBoundary(stringBoundary)) {
-            throw new IndexOutOfBoundsException(String.format("%s is out of bounds for %s", stringBoundary, this.boundary));
-        }
-
-        if (stringBoundary.end() == this.boundary.end()) {
-            return positions.subList(stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start()), positions.size());
-        }
-
-        return positions.subList(stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start()),
-                stringIdxToPositionIdx.get(stringBoundary.end() - this.boundary.start()));
-    }
-
-
-    public List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary) {
-
-        List<Rectangle2D> positionsPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList())
-                .stream()
-                .map(this::getPositions)
-                .map(RectangleTransformations::rectangleUnion)
-                .toList();
-
-        return List.of(EntityPosition.builder().rectanglePerLine(positionsPerLine).pageNode(page).build());
-    }
-
-
-    @Override
-    public String toString() {
-
-        return searchText;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/mapper/DocumentDataMapper.java

@@ -1,146 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.mapper;
-
-import java.awt.geom.Rectangle2D;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicTextBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.PageData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.TableOfContentsData;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableCellNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class DocumentDataMapper {
-
-    public DocumentData toDocumentData(DocumentGraph documentGraph) {
-
-        List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
-                .distinct()
-                .map(DocumentDataMapper::toAtomicTextBlockData)
-                .toList();
-
-        List<AtomicPositionBlockData> atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
-                .distinct()
-                .map(DocumentDataMapper::toAtomicPositionBlockData)
-                .toList();
-
-        List<PageData> pageData = documentGraph.getPages().stream().map(DocumentDataMapper::toPageData).toList();
-        TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
-        return DocumentData.builder()
-                .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
-                .atomicPositionBlocks(atomicPositionBlockData.toArray(new AtomicPositionBlockData[0]))
-                .pages(pageData.toArray(new PageData[0]))
-                .tableOfContents(tableOfContentsData)
-                .build();
-    }
-
-
-    private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) {
-
-        return new TableOfContentsData(toEntryData(tableOfContents.getRoot()));
-    }
-
-
-    private TableOfContentsData.EntryData toEntryData(TableOfContents.Entry entry) {
-
-        Long[] atomicTextBlocks;
-
-        if (entry.getNode().isTerminal()) {
-            atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getTerminalTextBlock());
-        } else {
-            atomicTextBlocks = new Long[]{};
-        }
-
-        Map<String, String> properties = switch (entry.getType()) {
-            case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.getNode());
-            case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.getNode());
-            case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.getNode());
-            default -> new HashMap<>();
-        };
-
-        return TableOfContentsData.EntryData.builder()
-                .tocId(toPrimitiveIntArray(entry.getTocId()))
-                .subEntries(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
-                .type(entry.getType())
-                .atomicBlocks(atomicTextBlocks)
-                .pages(entry.getNode().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new))
-                .properties(properties)
-                .build();
-    }
-
-
-    private Long[] toAtomicTextBlockIds(TextBlock textBlock) {
-
-        return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);
-    }
-
-
-    private PageData toPageData(PageNode p) {
-
-        return PageData.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
-    }
-
-
-    private AtomicTextBlockData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {
-
-        return AtomicTextBlockData.builder()
-                .id(atomicTextBlock.getId())
-                .page(atomicTextBlock.getPage().getNumber().longValue())
-                .searchText(atomicTextBlock.getSearchText())
-                .numberOnPage(atomicTextBlock.getNumberOnPage())
-                .start(atomicTextBlock.getBoundary().start())
-                .end(atomicTextBlock.getBoundary().end())
-                .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
-                .build();
-    }
-
-
-    private AtomicPositionBlockData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {
-
-        return AtomicPositionBlockData.builder()
-                .id(atomicTextBlock.getId())
-                .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
-                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
-                .build();
-    }
-
-
-    private float[][] toPrimitiveFloatMatrix(List<Rectangle2D> positions) {
-
-        float[][] positionMatrix = new float[positions.size()][];
-        for (int i = 0; i < positions.size(); i++) {
-            float[] singlePositions = new float[4];
-            singlePositions[0] = (float) positions.get(i).getMinX();
-            singlePositions[1] = (float) positions.get(i).getMinY();
-            singlePositions[2] = (float) positions.get(i).getWidth();
-            singlePositions[3] = (float) positions.get(i).getHeight();
-            positionMatrix[i] = singlePositions;
-        }
-        return positionMatrix;
-    }
-
-
-    private int[] toPrimitiveIntArray(List<Integer> list) {
-
-        int[] array = new int[list.size()];
-        for (int i = 0; i < list.size(); i++) {
-            array[i] = list.get(i);
-        }
-        return array;
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/mapper/DocumentGraphMapper.java

@@ -1,229 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.mapper;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType.FOOTER;
-import static com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType.HEADER;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-
-import com.google.common.primitives.Ints;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicTextBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.PageData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.TableOfContentsData;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.FooterNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.HeaderNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.HeadlineNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ParagraphNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SectionNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableCellNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlockCollector;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class DocumentGraphMapper {
-
-    public DocumentGraph toDocumentGraph(DocumentData documentData) {
-
-
-        DocumentGraph documentGraph = new DocumentGraph();
-        Context context = new Context(documentData,
-                new TableOfContents(documentGraph),
-                new LinkedList<>(),
-                new LinkedList<>(),
-                Arrays.stream(documentData.getAtomicTextBlocks()).toList(),
-                Arrays.stream(documentData.getAtomicPositionBlocks()).toList());
-
-        context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList());
-
-        context.tableOfContents.getRoot().getChildren().addAll(buildEntries(documentData.getTableOfContents().getRoot().getSubEntries(), context));
-
-        documentGraph.setTableOfContents(context.tableOfContents);
-        documentGraph.setPages(new HashSet<>(context.pages));
-        documentGraph.setNumberOfPages(documentData.getPages().length);
-
-        documentGraph.setTextBlock(documentGraph.buildTextBlock());
-        return documentGraph;
-    }
-
-
-    private List<TableOfContents.Entry> buildEntries(List<TableOfContentsData.EntryData> entries,
-                                                     Context context) {
-
-        List<TableOfContents.Entry> newEntries = new LinkedList<>();
-        for (TableOfContentsData.EntryData entryData : entries) {
-
-            boolean terminal = isTerminal(entryData);
-            List<PageNode> pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList();
-
-            SemanticNode node = switch (entryData.getType()) {
-                case SECTION -> buildSection(context);
-                case PARAGRAPH -> buildParagraph(context, terminal);
-                case HEADLINE -> buildHeadline(context, terminal);
-                case HEADER -> buildHeader(context, terminal);
-                case FOOTER -> buildFooter(context, terminal);
-                case TABLE -> buildTable(context, entryData.getProperties());
-                case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal);
-                case IMAGE -> buildImage(context, entryData.getProperties());
-                default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
-            };
-
-            if (node.isTerminal()) {
-                TextBlock textBlock = toTextBlock(entryData.getAtomicBlocks(), context, node);
-                node.setTerminalTextBlock(textBlock);
-            }
-            List<Integer> tocId = Arrays.stream(entryData.getTocId()).boxed().toList();
-            node.setTocId(tocId);
-
-            if (entryData.getType() == HEADER) {
-                pages.forEach(page -> page.setHeader((HeaderNode) node));
-            } else if (entryData.getType() == FOOTER) {
-                pages.forEach(page -> page.setFooter((FooterNode) node));
-            } else {
-                pages.forEach(page -> page.getMainBody().add(node));
-            }
-            newEntries.add(TableOfContents.Entry.builder().tocId(tocId).type(entryData.getType()).children(buildEntries(entryData.getSubEntries(), context)).node(node).build());
-        }
-        return newEntries;
-    }
-
-
-    private HeadlineNode buildHeadline(Context context, boolean terminal) {
-
-        return HeadlineNode.builder().terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private static boolean isTerminal(TableOfContentsData.EntryData entryData) {
-
-        return entryData.getAtomicBlocks().length > 0;
-    }
-
-
-    private ImageNode buildImage(Context context, Map<String, String> properties) {
-
-        var builder = ImageNode.builder();
-        PropertiesMapper.parseImageProperties(properties, builder);
-        return builder.tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private TableCellNode buildTableCell(Context context, Map<String, String> properties, boolean terminal) {
-
-        TableCellNode.TableCellNodeBuilder builder = TableCellNode.builder();
-        PropertiesMapper.parseTableCellProperties(properties, builder);
-        return builder.terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private TableNode buildTable(Context context, Map<String, String> properties) {
-
-        TableNode.TableNodeBuilder builder = TableNode.builder();
-        PropertiesMapper.parseTableProperties(properties, builder);
-        return TableNode.builder().tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private FooterNode buildFooter(Context context, boolean terminal) {
-
-        return FooterNode.builder().terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private HeaderNode buildHeader(Context context, boolean terminal) {
-
-        return HeaderNode.builder().terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private SectionNode buildSection(Context context) {
-
-        return SectionNode.builder().tableOfContents(context.tableOfContents()).build();
-
-    }
-
-
-    private ParagraphNode buildParagraph(Context context, boolean terminal) {
-
-        return ParagraphNode.builder().terminal(terminal).tableOfContents(context.tableOfContents()).build();
-    }
-
-
-    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
-
-        return Arrays.stream(atomicTextBlockIds)
-                .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                        context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
-                        parent,
-                        context))
-                .collect(new TextBlockCollector());
-    }
-
-
-    private PageNode buildPage(PageData p) {
-
-        return PageNode.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
-    }
-
-
-    private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData,
-                                              AtomicPositionBlockData atomicPositionBlockData,
-                                              SemanticNode parent,
-                                              Context context) {
-
-        return AtomicTextBlock.builder()
-                .id(atomicTextBlockData.getId())
-                .numberOnPage(atomicTextBlockData.getNumberOnPage())
-                .page(getPage(atomicTextBlockData.getPage(), context))
-                .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
-                .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks()))
-                .stringIdxToPositionIdx(Ints.asList(atomicPositionBlockData.getStringIdxToPositionIdx()))
-                .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
-                .parent(parent)
-                .build();
-    }
-
-
-    private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
-
-        return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList();
-    }
-
-
-    private PageNode getPage(Long pageIndex, Context context) {
-
-        return context.pages.stream()
-                .filter(page -> page.getNumber() == Math.toIntExact(pageIndex))
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException(String.format("ClassificationPage with number %d not found", pageIndex)));
-    }
-
-
-    record Context(
-            DocumentData layoutParsingModel,
-            TableOfContents tableOfContents,
-            List<PageNode> pages,
-            List<SectionNode> sections,
-            List<AtomicTextBlockData> atomicTextBlockData,
-            List<AtomicPositionBlockData> atomicPositionBlockData) {
-
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/mapper/PropertiesMapper.java

@@ -1,101 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.mapper;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableCellNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.services.RectangleTransformations;
-
-public class PropertiesMapper {
-
-    public static Map<String, String> buildImageProperties(ImageNode image) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put("imageType", image.getImageType().toString());
-        properties.put("transparency", String.valueOf(image.isTransparency()));
-        properties.put("position", RectangleTransformations.toString(image.getPosition()));
-        return properties;
-    }
-
-
-    public static Map<String, String> buildTableCellProperties(TableCellNode tableCell) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put("row", String.valueOf(tableCell.getRow()));
-        properties.put("col", String.valueOf(tableCell.getCol()));
-        properties.put("header", String.valueOf(tableCell.isHeader()));
-
-        if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
-            throw new IllegalArgumentException("TableCell can only occur on a single page!");
-        }
-        String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
-        properties.put("bBox", bBoxString);
-
-        return properties;
-    }
-
-
-    public static Map<String, String> buildTableProperties(TableNode table) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put("numberOfRows", String.valueOf(table.getNumberOfRows()));
-        properties.put("numberOfCols", String.valueOf(table.getNumberOfCols()));
-        return properties;
-    }
-
-
-    public static void parseImageProperties(Map<String, String> properties, ImageNode.ImageNodeBuilder builder) {
-
-        builder.imageType(parseImageType(properties.get("imageType")));
-        builder.transparency(Boolean.parseBoolean(properties.get("transparency")));
-        builder.position(parseRectangle2D(properties.get("position")));
-    }
-
-
-    public static void parseTableCellProperties(Map<String, String> properties, TableCellNode.TableCellNodeBuilder builder) {
-
-        builder.row(Integer.parseInt(properties.get("row")));
-        builder.col(Integer.parseInt(properties.get("col")));
-        builder.header(Boolean.parseBoolean(properties.get("header")));
-        builder.bBox(parseRectangle2D(properties.get("bBox")));
-    }
-
-
-    public static void parseTableProperties(Map<String, String> properties, TableNode.TableNodeBuilder builder) {
-
-        builder.numberOfRows(Integer.parseInt(properties.get("numberOfRows")));
-        builder.numberOfCols(Integer.parseInt(properties.get("numberOfCols")));
-    }
-
-
-    private static ImageType parseImageType(String imageType) {
-
-        return switch (imageType) {
-            case "LOGO" -> ImageType.LOGO;
-            case "FORMULA" -> ImageType.FORMULA;
-            case "SIGNATURE" -> ImageType.SIGNATURE;
-            case "OCR" -> ImageType.OCR;
-            default -> ImageType.OTHER;
-        };
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return String.format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "LAYOUTPARSING_REQUEST_QUEUE";
-    public static final String LAYOUT_PARSING_DLQ = "LAYOUTPARSING_DLQ";
-    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "LAYOUTPARSING_FINISHED_EVENT_QUEUE";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
+    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_dead_letter_queue";
+    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -4,17 +4,22 @@ import java.util.Map;
 import java.util.Optional;
 
 import lombok.Builder;
-
+import lombok.NonNull;
 
 @Builder
 public record LayoutParsingRequest(
+        @NonNull LayoutParsingType layoutParsingType,
         Map<String, String> identifier,
-        String originFileStorageId,
+        @NonNull String originFileStorageId,
         Optional<String> tablesFileStorageId,
         Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId) {
+        @NonNull String structureFileStorageId,
+        String researchDocumentStorageId,
+        @NonNull String textBlockFileStorageId,
+        @NonNull String positionBlockFileStorageId,
+        @NonNull String pageFileStorageId,
+        @NonNull String simplifiedTextStorageId,
+        @NonNull String viewerDocumentStorageId,
+        @NonNull String sectionGridStorageId) {
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java

@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.queue;
+
+public enum LayoutParsingType {
+    REDACT_MANAGER,
+    TAAS,
+    DOCUMINE
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/services/EntityEnrichmentService.java

@@ -1,10 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.services;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-public interface EntityEnrichmentService {
-
-    void enrichEntity(EntityNode entity, TextBlock textBlock);
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/services/EntityInsertionService.java

@@ -1,56 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.services;
-
-import java.util.Collections;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.entity.EntityNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-
-import lombok.RequiredArgsConstructor;
-
-@RequiredArgsConstructor
-public class EntityInsertionService {
-
-    private final EntityEnrichmentService entityEnrichmentService;
-
-
-    public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
-
-        try {
-            SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList())
-                    .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
-                    .findFirst()
-                    .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
-
-            containingNode.addThisToEntityIfIntersects(entity);
-
-            TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock();
-            entityEnrichmentService.enrichEntity(entity, textBlock);
-
-            addToPages(entity);
-            addToNodeEntitySets(entity);
-
-        } catch (NoSuchElementException e) {
-            entity.removeFromGraph();
-        }
-    }
-
-
-    private void addToPages(EntityNode entity) {
-
-        Set<PageNode> pages = entity.getDeepestFullyContainingNode().getPages();
-        entity.getPages().addAll(pages);
-        pages.forEach(page -> page.getEntities().add(entity));
-    }
-
-
-    private void addToNodeEntitySets(EntityNode entity) {
-
-        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/services/RectangleTransformations.java

@@ -1,95 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.services;
-
-import static java.lang.String.format;
-
-import java.awt.geom.Area;
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BinaryOperator;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collector;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class RectangleTransformations {
-
-    public static Rectangle2D pad(Rectangle2D rectangle2D, int deltaX, int deltaY) {
-
-        return new Rectangle2D.Double(rectangle2D.getMinX() - deltaX, rectangle2D.getMinY() - deltaY, rectangle2D.getWidth() + 2 * deltaX, rectangle2D.getHeight() + 2 * deltaY);
-    }
-
-
-    public static Rectangle2D bBoxUnionAtomicTextBlock(List<AtomicTextBlock> atomicTextBlocks) {
-
-        return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DUnion());
-    }
-
-
-    public static Rectangle2D rectangleUnion(List<Rectangle2D> rectangle2DList) {
-
-        return rectangle2DList.stream().collect(new Rectangle2DUnion());
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-
-    private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
-
-        @Override
-        public Supplier<Area> supplier() {
-
-            return Area::new;
-        }
-
-
-        @Override
-        public BiConsumer<Area, Rectangle2D> accumulator() {
-
-            return (area, rectangle2D) -> area.add(new Area(rectangle2D));
-        }
-
-
-        @Override
-        public BinaryOperator<Area> combiner() {
-
-            return (area1, area2) -> {
-                area1.add(area2);
-                return area1;
-            };
-        }
-
-
-        @Override
-        public Function<Area, Rectangle2D> finisher() {
-
-            return Area::getBounds2D;
-        }
-
-
-        @Override
-        public Set<Characteristics> characteristics() {
-
-            return Set.of(Characteristics.CONCURRENT, Characteristics.UNORDERED);
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-internal-api/src/test/resources/log4j2-test.xml

@@ -0,0 +1,16 @@
+<Configuration>
+
+    <Appenders>
+        <Console name="CONSOLE" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="warn">
+            <AppenderRef ref="CONSOLE"/>
+        </Root>
+        <Logger name="com.iqser" level="info"/>
+    </Loggers>
+
+</Configuration>

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -0,0 +1,26 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.2.2"
+}
+
+description = "layoutparser-service-processor"
+
+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0-RC1"
+
+dependencies {
+    implementation(project(":layoutparser-service-internal-api"))
+
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.126.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
+    }
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("com.iqser.red.commons:storage-commons:2.27.0")
+
+    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.1.2")
+}

layoutparser-service/layoutparser-service-processor/pom.xml

@@ -1,129 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>1.0.0</version>
-    </parent>
-
-    <artifactId>layoutparser-service-processor</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-internal-api-v1</artifactId>
-            <version>2.36.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>layoutparser-service-internal-api</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>storage-commons</artifactId>
-            <version>1.13.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>spring-commons</artifactId>
-            <version>6.2.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.dslplatform</groupId>
-            <artifactId>dsl-json-java8</artifactId>
-            <version>1.10.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox-tools</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>31.1-jre</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-afterburner</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-jsr310</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-security</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-web</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <optional>true</optional>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-starter-openfeign</artifactId>
-            <version>4.0.2</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-amqp</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
-    </dependencies>
-
-    <repositories>
-        <repository>
-            <id>spring-milestones</id>
-            <name>Spring Milestones</name>
-            <url>https://repo.spring.io/milestone</url>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-        </repository>
-        <repository>
-            <id>spring-snapshots</id>
-            <name>Spring Snapshots</name>
-            <url>https://repo.spring.io/snapshot</url>
-            <releases>
-                <enabled>false</enabled>
-            </releases>
-        </repository>
-    </repositories>
-    <pluginRepositories>
-        <pluginRepository>
-            <id>spring-milestones</id>
-            <name>Spring Milestones</name>
-            <url>https://repo.spring.io/milestone</url>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-        </pluginRepository>
-        <pluginRepository>
-            <id>spring-snapshots</id>
-            <name>Spring Snapshots</name>
-            <url>https://repo.spring.io/snapshot</url>
-            <releases>
-                <enabled>false</enabled>
-            </releases>
-        </pluginRepository>
-    </pluginRepositories>
-
-</project>

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -0,0 +1,234 @@
+package com.knecon.fforesight.service.layoutparser.processor;
+
+import static java.lang.String.format;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
+import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class LayoutParsingPipeline {
+
+    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
+    private final CvTableParsingAdapter cvTableParsingAdapter;
+    private final LayoutParsingStorageService layoutParsingStorageService;
+    private final SectionsBuilderService sectionsBuilderService;
+    private final SectionGridCreatorService sectionGridCreatorService;
+    private final TaasClassificationService taasClassificationService;
+    private final RedactManagerClassificationService redactManagerClassificationService;
+    private final DocuMineClassificationService docuMineClassificationService;
+    private final SimplifiedSectionTextService simplifiedSectionTextService;
+    private final BodyTextFrameService bodyTextFrameService;
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;
+    private final ViewerDocumentService viewerDocumentService;
+
+
+    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
+
+        long start = System.currentTimeMillis();
+
+        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
+            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            }
+
+            TableServiceResponse tableServiceResponse = new TableServiceResponse();
+            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+            }
+
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+
+            int numberOfPages = originDocument.getNumberOfPages();
+
+            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
+            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+            try (var out = new ByteArrayOutputStream()) {
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out);
+                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
+            }
+
+            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+            }
+
+            return LayoutParsingFinishedEvent.builder()
+                    .identifier(layoutParsingRequest.identifier())
+                    .numberOfPages(numberOfPages)
+                    .duration(System.currentTimeMillis() - start)
+                    .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
+                            layoutParsingRequest.structureFileStorageId(),
+                            layoutParsingRequest.textBlockFileStorageId(),
+                            layoutParsingRequest.positionBlockFileStorageId(),
+                            layoutParsingRequest.pageFileStorageId()))
+                    .build();
+        }
+    }
+
+
+    @SneakyThrows
+    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
+                                              PDDocument originDocument,
+                                              ImageServiceResponse imageServiceResponse,
+                                              TableServiceResponse tableServiceResponse) {
+
+        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
+        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
+
+        ClassificationDocument classificationDocument = new ClassificationDocument();
+        List<ClassificationPage> classificationPages = new ArrayList<>();
+
+        originDocument.setAllSecurityToBeRemoved(true);
+        long pageCount = originDocument.getNumberOfPages();
+
+        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+            classificationDocument.setPages(classificationPages);
+            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+            PDPage pdPage = originDocument.getPage(pageNumber - 1);
+            stripper.setPageNumber(pageNumber);
+            stripper.setStartPage(pageNumber);
+            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE)) {
+                stripper.setSortByPosition(true);
+            }
+            stripper.getText(originDocument);
+
+            PDRectangle pdr = pdPage.getMediaBox();
+
+            int rotation = pdPage.getRotation();
+            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+            PDRectangle cropbox = pdPage.getCropBox();
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                    stripper.getRulings(),
+                    stripper.getMinCharWidth(),
+                    stripper.getMaxCharHeight());
+
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            };
+            classificationPage.setCleanRulings(cleanRulings);
+            classificationPage.setRotation(rotation);
+            classificationPage.setLandscape(isLandscape);
+            classificationPage.setPageNumber(pageNumber);
+            classificationPage.setPageWidth(cropbox.getWidth());
+            classificationPage.setPageHeight(cropbox.getHeight());
+
+            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+                classificationPage.setImages(pdfImages.get(pageNumber));
+                imageServiceResponseAdapter.findOcr(classificationPage);
+            }
+
+            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            buildPageStatistics(classificationPage);
+            increaseDocumentStatistics(classificationPage, classificationDocument);
+
+            classificationPages.add(classificationPage);
+        }
+
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+
+        switch (layoutParsingType) {
+            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
+            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
+        }
+
+        sectionsBuilderService.buildSections(classificationDocument);
+        sectionsBuilderService.addImagesToSections(classificationDocument);
+
+        return classificationDocument;
+    }
+
+
+    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
+
+        if (!classificationPage.isLandscape()) {
+            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        }
+        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
+        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
+        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
+    }
+
+
+    private void buildPageStatistics(ClassificationPage classificationPage) {
+
+        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
+        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
+                    continue;
+                }
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
+                    classificationPage.getFontCounter().add(word.getFont());
+                    classificationPage.getFontSizeCounter().add(word.getFontSize());
+                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
+                }
+            }
+        }
+
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingService.java

@@ -1,88 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor;
-
-import static java.lang.String.format;
-
-import java.io.IOException;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
-import com.knecon.fforesight.service.layoutparser.internal.api.mapper.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.ClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.PdfParsingService;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.SectionsBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class LayoutParsingService {
-
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final CvTableParsingAdapter cvTableParsingAdapter;
-    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final PdfParsingService pdfParsingService;
-    private final ClassificationService classificationService;
-    private final SectionsBuilderService sectionsBuilderService;
-    private final DocumentGraphFactory documentGraphFactory;
-
-
-    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
-
-        long start = System.currentTimeMillis();
-        PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-
-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
-        }
-
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
-        }
-
-        DocumentGraph documentGraph = parseLayout(originDocument, imageServiceResponse, tableServiceResponse);
-        int numberOfPages = originDocument.getNumberOfPages();
-        originDocument.close();
-
-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-
-        return LayoutParsingFinishedEvent.builder()
-                .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(numberOfPages)
-                .duration(System.currentTimeMillis() - start)
-                .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId()))
-                .build();
-    }
-
-
-    public DocumentGraph parseLayout(PDDocument originDocument, ImageServiceResponse imageServiceResponse, TableServiceResponse tableServiceResponse) {
-
-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
-
-        classificationService.classifyDocument(classificationDocument);
-
-        sectionsBuilderService.buildSections(classificationDocument);
-
-        return documentGraphFactory.buildDocumentGraph(classificationDocument);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -1,10 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
@@ -13,18 +18,18 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicTextBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.PageData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.TableOfContentsData;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.multitenancy.TenantContext;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -38,10 +43,11 @@ public class LayoutParsingStorageService {
 
     public PDDocument getOriginFile(String storageId) throws IOException {
 
-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var originDocumentInputStream = getObject(storageId)) {
             File tempFile = createTempFile("document", ".pdf");
             try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
                 IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
             }
             return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
         }
@@ -50,52 +56,44 @@ public class LayoutParsingStorageService {
 
     public ImageServiceResponse getImagesFile(String storageId) throws IOException {
 
-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (InputStream inputStream = getObject(storageId)) {
 
-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
         }
     }
 
 
     public TableServiceResponse getTablesFile(String storageId) throws IOException {
 
-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+        try (var tableClassificationStream = getObject(storageId)) {
 
+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
         }
     }
 
 
-    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) throws IOException {
-
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getTableOfContents());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getAtomicTextBlocks());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getAtomicPositionBlocks());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getPages());
+    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {
 
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getDocumentTextData());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getDocumentPositions());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getDocumentPages());
     }
 
 
-    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
+    public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) {
 
-        PageData[] pageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), PageData[].class);
-        AtomicTextBlockData[] atomicTextBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.textBlockFileStorageId(),
-                AtomicTextBlockData[].class);
-        AtomicPositionBlockData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.positionBlockFileStorageId(),
-                AtomicPositionBlockData[].class);
-        TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.structureFileStorageId(),
-                TableOfContentsData.class);
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid);
+    }
 
-        return DocumentData.builder()
-                .tableOfContents(tableOfContentsData)
-                .atomicPositionBlocks(atomicPositionBlockData)
-                .atomicTextBlocks(atomicTextBlockData)
-                .pages(pageData)
-                .build();
+
+    public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.researchDocumentStorageId(), researchDocumentData);
     }
 
 
@@ -123,4 +121,30 @@ public class LayoutParsingStorageService {
         }
     }
 
+
+    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
+    }
+
+
+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
+
+    @SneakyThrows
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {
+
+        try (var in = new ByteArrayInputStream(out.toByteArray())) {
+
+            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
+        }
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/CvTableParsingAdapter.java

@@ -1,49 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.CvParsedTableCell;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class CvTableParsingAdapter {
-
-    public Map<Integer, List<com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell>> buildCvParsedTablesPerPage(TableServiceResponse tableServiceResponse) {
-
-        Map<Integer, List<com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell>> tableCells = new HashMap<>();
-        tableServiceResponse.getData()
-                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells())));
-
-        return tableCells;
-    }
-
-
-    private Collection<? extends com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell> convertTableCells(List<CvParsedTableCell> tableCells) {
-
-        List<com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell> cvParsedTableCells = new ArrayList<>();
-
-        tableCells.forEach(t -> cvParsedTableCells.add(com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell.builder()
-                .y0(t.getY0())
-                .x1(t.getX1())
-                .y1(t.getY1())
-                .x0(t.getX0())
-                .width(t.getWidth())
-                .height(t.getHeight())
-                .build()));
-
-        return cvParsedTableCells;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Classification.java

@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Classification {
-
-    private Map<String, Float> probabilities = new HashMap<>();
-    private String label;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/FilterGeometry.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class FilterGeometry {
-
-    private ImageSize imageSize;
-    private Format imageFormat;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Filters.java

@@ -1,15 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Filters {
-
-    private FilterGeometry geometry;
-    private Probability probability;
-    private boolean allPassed;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Format.java

@@ -1,15 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Format {
-
-    private float quotient;
-    private boolean tooTall;
-    private boolean tooWide;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Geometry.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Geometry {
-
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageServiceResponse.java

@@ -1,33 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonAlias;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class ImageServiceResponse {
-
-    private String dossierId;
-    private String fileId;
-
-    @JsonProperty(value = "imageMetadata")
-    @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
-    private List<Metadata> data = new ArrayList<>();
-
-    private List<Metadata> dataCV = new ArrayList<>();
-
-
-    @JsonProperty(value = "imageMetadata")
-    @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
-    public void setData(List<Metadata> data) {this.data = data;}
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageSize.java

@@ -1,15 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class ImageSize {
-
-    private float quotient;
-    private boolean tooLarge;
-    private boolean tooSmall;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Metadata.java

@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Metadata {
-
-    private Classification classification;
-    private Position position;
-    private Geometry geometry;
-    private Filters filters;
-    private boolean alpha;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Position.java

@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Position {
-
-    private float x1;
-    private float x2;
-    private float y1;
-    private float y2;
-    private int pageNumber;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Probability.java

@@ -1,13 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class Probability {
-
-    private boolean unconfident;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/CvParsedPageInfo.java

@@ -1,16 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class CvParsedPageInfo {
-
-    private int number;
-    private int rotation;
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/CvParsedTableCell.java

@@ -1,18 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class CvParsedTableCell {
-
-    private float x0;
-    private float y0;
-    private float x1;
-    private float y1;
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/CvParsedTableModel.java

@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.Data;
-
-@Data
-@CompiledJson
-public class CvParsedTableModel {
-
-    private CvParsedPageInfo pageInfo;
-    private List<CvParsedTableCell> tableCells = new ArrayList<>();
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/AbstractTextContainer.java

@@ -1,71 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
-
-import java.awt.geom.Rectangle2D;
-
-import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextBlockOrientation;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public abstract class AbstractTextContainer {
-
-    protected float minX;
-    protected float maxX;
-    protected float minY;
-    protected float maxY;
-    protected String classification;
-    protected int page;
-
-    private TextBlockOrientation orientation = TextBlockOrientation.NONE;
-
-
-    public abstract String getText();
-
-
-    public boolean containsBlock(ClassificationTextBlock other) {
-
-        return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY();
-    }
-
-
-    public boolean contains(AbstractTextContainer other) {
-
-        return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
-    }
-
-
-    public boolean contains(Rectangle2D other) {
-
-        return other.contains(minX, minY, getWidth(), getHeight());
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getHeight() {
-
-        return maxY - minY;
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getWidth() {
-
-        return maxX - minX;
-    }
-
-
-    public boolean intersectsY(AbstractTextContainer atc) {
-
-        return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/ClassificationFooter.java

@@ -1,16 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
-
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-@Data
-@AllArgsConstructor
-public class ClassificationFooter {
-
-    private List<ClassificationTextBlock> textBlocks;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/ClassificationHeader.java

@@ -1,16 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
-
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-@Data
-@AllArgsConstructor
-public class ClassificationHeader {
-
-    private List<ClassificationTextBlock> textBlocks;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/ClassificationSection.java

@@ -1,38 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.Table;
-
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-public class ClassificationSection implements Comparable {
-
-    private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
-    private List<ClassifiedImage> images = new ArrayList<>();
-    private String headline;
-
-
-    public List<Table> getTables() {
-
-        List<Table> tables = new ArrayList<>();
-        pageBlocks.forEach(block -> {
-            if (block instanceof Table) {
-                tables.add((Table) block);
-            }
-        });
-        return tables;
-    }
-
-
-    @Override
-    public int compareTo(Object o) {
-
-        return 0;
-    }
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/table/TableCell.java

@@ -1,38 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto.table;
-
-import java.awt.geom.Point2D;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.Rectangle;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
-@SuppressWarnings("serial")
-@Data
-@EqualsAndHashCode(callSuper = true)
-@NoArgsConstructor
-public class TableCell extends Rectangle {
-
-    private List<ClassificationTextBlock> textBlocks = new ArrayList<>();
-
-    private List<TableCell> headerCells = new ArrayList<>();
-
-    private boolean isHeaderCell;
-
-
-    public TableCell(Point2D topLeft, Point2D bottomRight) {
-
-        super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()));
-    }
-
-
-    public void addTextBlock(ClassificationTextBlock textBlock) {
-
-        textBlocks.add(textBlock);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/dto/text/TextBlockOrientation.java

@@ -1,8 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto.text;
-
-public enum TextBlockOrientation {
-
-    NONE,
-    LEFT,
-    RIGHT
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/parsing/PDFAreaTextStripper.java

@@ -1,82 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.parsing;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.pdfbox.text.PDFTextStripperByArea;
-import org.apache.pdfbox.text.TextPosition;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-import lombok.Getter;
-import lombok.Setter;
-
-public class PDFAreaTextStripper extends PDFTextStripperByArea {
-
-    @Getter
-    private List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-
-    @Setter
-    private int pageNumber;
-
-
-    public PDFAreaTextStripper() throws IOException {
-
-    }
-
-
-    @Override
-    public void writeString(String text, List<TextPosition> textPositions) throws IOException {
-
-        int startIndex = 0;
-        for (int i = 0; i <= textPositions.size() - 1; i++) {
-
-            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) {
-                startIndex++;
-                continue;
-            }
-
-            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i;
-            }
-
-            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i;
-            }
-
-            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i + 1;
-            }
-        }
-
-        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
-            sublist = sublist.subList(0, sublist.size() - 1);
-        }
-        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-            textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-        }
-        super.writeString(text);
-    }
-
-
-    public void clearPositions() {
-
-        textPositionSequences = new ArrayList<>();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/service/BodyTextFrameService.java

@@ -1,160 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.service;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.FloatFrequencyCounter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.AbstractTextContainer;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.Table;
-import com.knecon.fforesight.service.layoutparser.processor.classification.utils.PositionUtils;
-
-@Service
-public class BodyTextFrameService {
-
-    /**
-     * Adjusts and sets the body text frame to a classificationPage.
-     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the classificationPage rotation.
-     * 0 -> LowerLeft
-     * 90 -> UpperLeft
-     * 180 -> UpperRight
-     * 270 -> LowerRight
-     * The aspect ratio of the classificationPage is also regarded.
-     *
-     * @param classificationPage                   The classificationPage
-     * @param bodyTextFrame          frame that contains the main text on portrait pages
-     * @param landscapeBodyTextFrame frame that contains the main text on landscape pages
-     */
-    public void setBodyTextFrameAdjustedToPage(ClassificationPage classificationPage, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
-
-        Rectangle textFrame = classificationPage.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
-
-        if (classificationPage.getPageWidth() > classificationPage.getPageHeight() && classificationPage.getRotation() == 270) {
-            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getY(), classificationPage.getPageHeight() - textFrame.getTopLeft().getX() - textFrame.getWidth()),
-                    textFrame.getHeight(),
-                    textFrame.getWidth(),
-                    0);
-        } else if (classificationPage.getPageWidth() > classificationPage.getPageHeight() && classificationPage.getRotation() != 0) {
-            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getY(), textFrame.getTopLeft().getX()), textFrame.getHeight(), textFrame.getWidth(), classificationPage.getPageNumber());
-        } else if (classificationPage.getRotation() == 180) {
-            textFrame = new Rectangle(new Point(textFrame.getTopLeft().getX(), classificationPage.getPageHeight() - textFrame.getTopLeft().getY() - textFrame.getHeight()),
-                    textFrame.getWidth(),
-                    textFrame.getHeight(),
-                    0);
-        }
-        classificationPage.setBodyTextFrame(textFrame);
-    }
-
-
-    /**
-     * Calculates the frame that contains the main text, text outside the frame will be e.g. headers or footers.
-     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
-     * 0 -> LowerLeft
-     * 90 -> UpperLeft
-     * 180 -> UpperRight
-     * 270 -> LowerRight
-     * The aspect ratio of the page is also regarded.
-     *
-     * @param classificationPages                   List of all classificationPages
-     * @param documentFontSizeCounter Statistics of the document
-     * @param landscape               Calculate for landscape or portrait
-     * @return Rectangle of the text frame
-     */
-    public Rectangle calculateBodyTextFrame(List<ClassificationPage> classificationPages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
-
-        BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();
-
-        for (ClassificationPage classificationPage : classificationPages) {
-
-            if (classificationPage.getTextBlocks().isEmpty() || landscape != classificationPage.isLandscape()) {
-                continue;
-            }
-
-            for (AbstractTextContainer container : classificationPage.getTextBlocks()) {
-
-                if (container instanceof ClassificationTextBlock) {
-                    ClassificationTextBlock textBlock = (ClassificationTextBlock) container;
-                    if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
-                        continue;
-                    }
-
-                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (approxLineCount < 2.9f) {
-                        continue;
-                    }
-
-                    if (documentFontSizeCounter.getMostPopular() != null && textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
-
-                        expandRectangle(textBlock, classificationPage, expansionsRectangle);
-                    }
-                }
-
-                if (container instanceof Table) {
-                    Table table = (Table) container;
-                    for (List<TableCell> row : table.getRows()) {
-                        for (TableCell cell : row) {
-
-                            if (cell == null || cell.getTextBlocks() == null) {
-                                continue;
-                            }
-                            for (ClassificationTextBlock textBlock : cell.getTextBlocks()) {
-                                expandRectangle(textBlock, classificationPage, expansionsRectangle);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        return new Rectangle(new Point(expansionsRectangle.minX, expansionsRectangle.minY),
-                expansionsRectangle.maxX - expansionsRectangle.minX,
-                expansionsRectangle.maxY - expansionsRectangle.minY,
-                0);
-    }
-
-
-    private void expandRectangle(ClassificationTextBlock textBlock, ClassificationPage classificationPage, BodyTextFrameExpansionsRectangle expansionsRectangle) {
-
-        if (classificationPage.getPageWidth() > classificationPage.getPageHeight() && classificationPage.getRotation() != 0) {
-            if (textBlock.getPdfMinY() < expansionsRectangle.minX) {
-                expansionsRectangle.minX = textBlock.getPdfMinY();
-            }
-            if (textBlock.getPdfMaxY() > expansionsRectangle.maxX) {
-                expansionsRectangle.maxX = textBlock.getPdfMaxY();
-            }
-            if (textBlock.getPdfMinX() < expansionsRectangle.minY) {
-                expansionsRectangle.minY = textBlock.getPdfMinX();
-            }
-            if (textBlock.getPdfMaxX() > expansionsRectangle.maxY) {
-                expansionsRectangle.maxY = textBlock.getPdfMaxX();
-            }
-        } else {
-            if (textBlock.getPdfMinX() < expansionsRectangle.minX) {
-                expansionsRectangle.minX = textBlock.getPdfMinX();
-            }
-            if (textBlock.getPdfMaxX() > expansionsRectangle.maxX) {
-                expansionsRectangle.maxX = textBlock.getPdfMaxX();
-            }
-            if (textBlock.getPdfMinY() < expansionsRectangle.minY) {
-                expansionsRectangle.minY = textBlock.getPdfMinY();
-            }
-            if (textBlock.getPdfMaxY() > expansionsRectangle.maxY) {
-                expansionsRectangle.maxY = textBlock.getPdfMaxY();
-            }
-        }
-    }
-
-
-    private class BodyTextFrameExpansionsRectangle {
-
-        float minX = 10000;
-        float maxX = -100;
-        float minY = 10000;
-        float maxY = -100;
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/service/PdfParsingService.java

@@ -1,134 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.service;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.parsing.PDFLinesTextStripper;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.AbstractTextContainer;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.CvParsedTableCell;
-
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class PdfParsingService {
-
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final BlockificationService blockificationService;
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-
-
-    public ClassificationDocument parseDocument(PDDocument originDocument, Map<Integer, List<CvParsedTableCell>> pdfTableCells, Map<Integer, List<ClassifiedImage>> pdfImages) {
-
-        ClassificationDocument document = new ClassificationDocument();
-        List<ClassificationPage> classificationPages = new ArrayList<>();
-
-        originDocument.setAllSecurityToBeRemoved(true);
-        long pageCount = originDocument.getNumberOfPages();
-
-        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-            parsePage(pdfImages, originDocument, pdfTableCells, document, classificationPages, pageNumber);
-        }
-
-        document.setPages(classificationPages);
-
-        return document;
-    }
-
-
-    @SneakyThrows
-    private void parsePage(Map<Integer, List<ClassifiedImage>> pdfImages,
-                           PDDocument pdDocument,
-                           Map<Integer, List<CvParsedTableCell>> pdfTableCells,
-                           ClassificationDocument document,
-                           List<ClassificationPage> classificationPages,
-                           int pageNumber) {
-
-        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-        stripper.setPageNumber(pageNumber);
-        stripper.setStartPage(pageNumber);
-        stripper.setEndPage(pageNumber);
-        stripper.setPdpage(pdPage);
-        stripper.getText(pdDocument);
-
-        PDRectangle pdr = pdPage.getMediaBox();
-
-        int rotation = pdPage.getRotation();
-        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-        PDRectangle cropbox = pdPage.getCropBox();
-        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                stripper.getRulings(),
-                stripper.getMinCharWidth(),
-                stripper.getMaxCharHeight());
-        ClassificationPage classificationPage = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-        classificationPage.setRotation(rotation);
-        classificationPage.setLandscape(isLandscape);
-        classificationPage.setPageNumber(pageNumber);
-        classificationPage.setPageWidth(cropbox.getWidth());
-        classificationPage.setPageHeight(cropbox.getHeight());
-
-        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-            classificationPage.setImages(pdfImages.get(pageNumber));
-            imageServiceResponseAdapter.findOcr(classificationPage);
-        }
-
-        tableExtractionService.removeRedundantTableCells(cleanRulings, classificationPage);
-        buildPageStatistics(classificationPage);
-        increaseDocumentStatistics(classificationPage, document);
-
-        classificationPages.add(classificationPage);
-    }
-
-
-    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
-
-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-        }
-        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
-        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
-        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
-    }
-
-
-    private void buildPageStatistics(ClassificationPage classificationPage) {
-
-        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
-        for (AbstractTextContainer textBlock : classificationPage.getTextBlocks()) {
-            if (textBlock instanceof ClassificationTextBlock) {
-                if (((ClassificationTextBlock) textBlock).getSequences() == null) {
-                    continue;
-                }
-                for (TextPositionSequence word : ((ClassificationTextBlock) textBlock).getSequences()) {
-                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
-                    classificationPage.getFontCounter().add(word.getFont());
-                    classificationPage.getFontSizeCounter().add(word.getFontSize());
-                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
-                }
-            }
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/DocumentGraphFactory.java

@@ -1,388 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import static java.lang.String.format;
-import static java.util.stream.Collectors.groupingBy;
-import static java.util.stream.Collectors.toList;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.DocumentGraph;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.FooterNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.HeaderNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.HeadlineNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ImageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.ParagraphNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SectionNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableCellNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.TableNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.AbstractTextContainer;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationFooter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationHeader;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.Table;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.table.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-
-@Service
-public class DocumentGraphFactory {
-
-    public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
-
-
-    public DocumentGraph buildDocumentGraph(ClassificationDocument document) {
-
-        TextBlockFactory textBlockFactory = new TextBlockFactory();
-        DocumentGraph documentGraph = new DocumentGraph();
-        Context context = new Context(new TableOfContents(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
-
-        document.getPages().stream().map(this::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
-        document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image));
-        addSections(document, context);
-        addHeaderAndFooterToEachPage(document, context);
-
-        documentGraph.setNumberOfPages(context.pages.size());
-        documentGraph.setPages(context.pages.keySet());
-        documentGraph.setTableOfContents(context.tableOfContents);
-        documentGraph.setTextBlock(documentGraph.buildTextBlock());
-        return documentGraph;
-    }
-
-
-    private void addSections(ClassificationDocument document, Context context) {
-
-        document.getSections().forEach(section -> addSection(null, section.getPageBlocks(), section.getImages(), context));
-    }
-
-
-    private void addSection(SemanticNode parentNode, List<AbstractTextContainer> pageBlocks, List<ClassifiedImage> images, Context context) {
-
-        Map<Integer, List<AbstractTextContainer>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractTextContainer::getPage));
-        SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build();
-
-        context.sections().add(sectionNode);
-        blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber));
-
-        List<Integer> tocId;
-        if (parentNode == null) {
-            tocId = context.tableOfContents.createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
-        } else {
-            tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
-        }
-        sectionNode.setTocId(tocId);
-        Set<AbstractTextContainer> alreadyMerged = new HashSet<>();
-        for (AbstractTextContainer abstractTextContainer : pageBlocks) {
-
-            if (alreadyMerged.contains(abstractTextContainer)) {
-                continue;
-            }
-
-            if (abstractTextContainer instanceof ClassificationTextBlock) {
-                List<ClassificationTextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractTextContainer, pageBlocks);
-                alreadyMerged.addAll(textBlocks);
-                addParagraphOrHeadline(sectionNode, (ClassificationTextBlock) abstractTextContainer, context, textBlocks);
-            }
-            if (abstractTextContainer instanceof Table) {
-                addTable(sectionNode, (Table) abstractTextContainer, context);
-            }
-        }
-        for (ClassifiedImage image : images) {
-
-            addImage(sectionNode, image, context);
-        }
-    }
-
-
-    private static List<ClassificationTextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) {
-
-        return pageBlocks.stream()
-                .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
-                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
-                .filter(abstractTextContainer -> abstractTextContainer instanceof ClassificationTextBlock)
-                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
-                .map(abstractTextContainer -> (ClassificationTextBlock) abstractTextContainer)
-                .toList();
-    }
-
-
-    private void addSectionNodeToPageNode(Context context, SectionNode sectionNode, Integer pageNumber) {
-
-        PageNode page = getPage(pageNumber, context);
-        page.getMainBody().add(sectionNode);
-    }
-
-
-    private void addTable(SemanticNode parentNode, Table table, Context context) {
-
-        PageNode page = getPage(table.getPage(), context);
-        TableNode tableNode = TableNode.builder().tableOfContents(context.tableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build();
-
-        if (!page.getMainBody().contains(parentNode)) {
-            parentNode.getPages().add(page);
-        }
-
-        page.getMainBody().add(tableNode);
-
-        List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE, tableNode);
-        tableNode.setTocId(tocId);
-
-        addTableCells(table.getRows(), tableNode, context, table.getPage());
-    }
-
-
-    private void addTableCells(List<List<TableCell>> rows, SemanticNode parentNode, Context context, int pageNumber) {
-
-        for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
-            for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, parentNode, pageNumber, context);
-            }
-        }
-    }
-
-
-    private void addTableCell(TableCell cell, int rowIndex, int colIndex, SemanticNode parentNode, int pageNumber, Context context) {
-
-        PageNode page = getPage(pageNumber, context);
-        cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(pageNumber));
-
-        TableCellNode tableCellNode = TableCellNode.builder()
-                .tableOfContents(context.tableOfContents())
-                .row(rowIndex)
-                .col(colIndex)
-                .header(cell.isHeaderCell())
-                .bBox(cell.getBounds2D())
-                .build();
-        page.getMainBody().add(tableCellNode);
-
-        TextBlock textBlock;
-
-        List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode);
-        tableCellNode.setTocId(tocId);
-
-        if (cell.getTextBlocks().isEmpty()) {
-            tableCellNode.setTerminalTextBlock(context.textBlockFactory.emptyTextBlock(parentNode, context, page));
-            tableCellNode.setTerminal(true);
-
-        } else if (cell.getTextBlocks().size() == 1) {
-            textBlock = context.textBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCellNode, context, page);
-            tableCellNode.setTerminalTextBlock(textBlock);
-            tableCellNode.setTerminal(true);
-
-        } else if (firstTextBlockIsHeadline(cell)) {
-            addSection(tableCellNode, cell.getTextBlocks().stream().map(tb -> (AbstractTextContainer) tb).toList(), Collections.emptyList(), context);
-            tableCellNode.setTerminal(false);
-
-        } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
-            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
-            textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page);
-            tableCellNode.setTerminalTextBlock(textBlock);
-            tableCellNode.setTerminal(true);
-
-        } else {
-            cell.getTextBlocks().forEach(tb -> addParagraphOrHeadline(tableCellNode, tb, context));
-            tableCellNode.setTerminal(false);
-        }
-
-    }
-
-
-    private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(TableCell cell, PageNode page) {
-
-        return cell.getArea() < TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
-    }
-
-
-    private static boolean firstTextBlockIsHeadline(TableCell cell) {
-
-        String classification = cell.getTextBlocks().get(0).getClassification();
-        return classification != null && classification.startsWith("H");
-    }
-
-
-    private void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context) {
-
-        addParagraphOrHeadline(parentNode, originalTextBlock, context, Collections.emptyList());
-    }
-
-
-    private void addParagraphOrHeadline(SemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context, List<ClassificationTextBlock> textBlocksToMerge) {
-
-        PageNode page = getPage(originalTextBlock.getPage(), context);
-
-        SemanticNode node;
-        if (originalTextBlock.getClassification() != null && originalTextBlock.getClassification().startsWith("H")) {
-            node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build();
-        } else {
-            node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build();
-        }
-
-        page.getMainBody().add(node);
-
-        List<ClassificationTextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
-        textBlocks.add(originalTextBlock);
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
-
-        if (node instanceof HeadlineNode headlineNode) {
-            List<Integer> tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.HEADLINE, node);
-            headlineNode.setTerminalTextBlock(textBlock);
-            headlineNode.setTocId(tocId);
-        }
-        if (node instanceof ParagraphNode paragraphNode) {
-            List<Integer> tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.PARAGRAPH, node);
-            paragraphNode.setTerminalTextBlock(textBlock);
-            paragraphNode.setTocId(tocId);
-        }
-    }
-
-
-    private void addImage(SectionNode sectionNode, ClassifiedImage image, Context context) {
-
-        PageNode page = getPage(image.getPage(), context);
-        ImageNode imageNode = ImageNode.builder()
-                .imageType(image.getImageType())
-                .position(image.getPosition())
-                .transparency(image.isHasTransparency())
-                .page(page)
-                .tableOfContents(context.tableOfContents())
-                .build();
-        page.getMainBody().add(imageNode);
-
-        List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(sectionNode.getTocId(), NodeType.IMAGE, imageNode);
-        imageNode.setTocId(tocId);
-    }
-
-
-    private void addHeaderAndFooterToEachPage(ClassificationDocument document, Context context) {
-
-        Map<Integer, List<ClassificationTextBlock>> headers = document.getHeaders()
-                .stream()
-                .map(ClassificationHeader::getTextBlocks)
-                .flatMap(List::stream)
-                .collect(groupingBy(AbstractTextContainer::getPage, toList()));
-
-        Map<Integer, List<ClassificationTextBlock>> footers = document.getFooters()
-                .stream()
-                .map(ClassificationFooter::getTextBlocks)
-                .flatMap(List::stream)
-                .collect(groupingBy(AbstractTextContainer::getPage, toList()));
-
-        for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) {
-            if (headers.containsKey(pageIndex)) {
-                addHeader(headers.get(pageIndex), context);
-            } else {
-                addEmptyHeader(pageIndex, context);
-            }
-        }
-
-        for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) {
-            if (footers.containsKey(pageIndex)) {
-                addFooter(footers.get(pageIndex), context);
-            } else {
-                addEmptyFooter(pageIndex, context);
-            }
-        }
-    }
-
-
-    private void addFooter(List<ClassificationTextBlock> textBlocks, Context context) {
-
-        PageNode page = getPage(textBlocks.get(0).getPage(), context);
-        FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
-                footer,
-                context,
-                page);
-        List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
-        footer.setTocId(tocId);
-        footer.setTerminalTextBlock(textBlock);
-        page.setFooter(footer);
-    }
-
-
-    public void addHeader(List<ClassificationTextBlock> textBlocks, Context context) {
-
-        PageNode page = getPage(textBlocks.get(0).getPage(), context);
-        HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
-                header,
-                context,
-                0,
-                page);
-        List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
-        header.setTocId(tocId);
-        header.setTerminalTextBlock(textBlock);
-        page.setHeader(header);
-    }
-
-
-    private void addEmptyFooter(int pageIndex, Context context) {
-
-        PageNode page = getPage(pageIndex, context);
-        FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page);
-        List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
-        footer.setTocId(tocId);
-        footer.setTerminalTextBlock(textBlock);
-        page.setFooter(footer);
-    }
-
-
-    private void addEmptyHeader(int pageIndex, Context context) {
-
-        PageNode page = getPage(pageIndex, context);
-        HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
-        List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
-        header.setTocId(tocId);
-        header.setTerminalTextBlock(textBlock);
-        page.setHeader(header);
-    }
-
-
-    private PageNode buildPage(ClassificationPage p) {
-
-        return PageNode.builder()
-                .height((int) p.getPageHeight())
-                .width((int) p.getPageWidth())
-                .number(p.getPageNumber())
-                .rotation(p.getRotation())
-                .mainBody(new LinkedList<>())
-                .build();
-    }
-
-
-    private PageNode getPage(int pageIndex, Context context) {
-
-        return context.pages.keySet()
-                .stream()
-                .filter(page -> page.getNumber() == pageIndex)
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException(format("ClassificationPage with number %d not found", pageIndex)));
-    }
-
-
-    record Context(
-            TableOfContents tableOfContents, Map<PageNode, AtomicInteger> pages, List<SectionNode> sections, List<ClassifiedImage> images, TextBlockFactory textBlockFactory) {
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/RectangleTransformations.java

@@ -1,105 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import static java.lang.String.format;
-
-import java.awt.geom.Area;
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BinaryOperator;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collector;
-
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.AbstractTextContainer;
-
-public class RectangleTransformations {
-
-    public static Rectangle2D pad(Rectangle2D rectangle2D, int deltaX, int deltaY) {
-
-        return new Rectangle2D.Double(rectangle2D.getMinX() - deltaX, rectangle2D.getMinY() - deltaY, rectangle2D.getWidth() + 2 * deltaX, rectangle2D.getHeight() + 2 * deltaY);
-    }
-
-
-    public static Rectangle2D bBoxUnionAbstractTextContainer(List<AbstractTextContainer> abstractTextContainers) {
-
-        return abstractTextContainers.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion());
-    }
-
-    public static Rectangle2D rectangleUnion(List<Rectangle2D> rectangle2DList) {
-
-        return rectangle2DList.stream().collect(new Rectangle2DUnion());
-    }
-
-
-    public static Rectangle2D toRectangle2D(AbstractTextContainer abstractTextContainer) {
-
-        return new Rectangle2D.Float(abstractTextContainer.getMinX(), abstractTextContainer.getMinY(), abstractTextContainer.getWidth(), abstractTextContainer.getHeight());
-    }
-
-
-    public static Rectangle2D toRectangle2D(PDRectangle rectangle) {
-
-        return new Rectangle2D.Double(rectangle.getLowerLeftX(), rectangle.getLowerLeftY(), rectangle.getWidth(), rectangle.getHeight());
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-
-    private static class Rectangle2DUnion implements Collector<Rectangle2D, Area, Rectangle2D> {
-
-        @Override
-        public Supplier<Area> supplier() {
-
-            return Area::new;
-        }
-
-
-        @Override
-        public BiConsumer<Area, Rectangle2D> accumulator() {
-
-            return (area, rectangle2D) -> area.add(new Area(rectangle2D));
-        }
-
-
-        @Override
-        public BinaryOperator<Area> combiner() {
-
-            return (area1, area2) -> {
-                area1.add(area2);
-                return area1;
-            };
-        }
-
-
-        @Override
-        public Function<Area, Rectangle2D> finisher() {
-
-            return Area::getBounds2D;
-        }
-
-
-        @Override
-        public Set<Characteristics> characteristics() {
-
-            return Set.of(Characteristics.CONCURRENT, Characteristics.UNORDERED);
-        }
-
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/SearchTextWithTextPositionFactory.java

@@ -1,156 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.awt.geom.AffineTransform;
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextDirection;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-public class SearchTextWithTextPositionFactory {
-
-    public static final int HEIGHT_PADDING = 2;
-
-
-    public static SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
-
-        if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
-            return SearchTextWithTextPositionModel.builder()
-                    .searchText("")
-                    .lineBreaks(Collections.emptyList())
-                    .positions(Collections.emptyList())
-                    .stringCoordsToPositionCoords(Collections.emptyList())
-                    .build();
-        }
-
-        List<Integer> stringIdxToPositionIdx = new LinkedList<>();
-        List<Integer> lineBreaksStringIdx = new LinkedList<>();
-        StringBuilder sb = new StringBuilder();
-
-        int stringIdx = 0;
-        int positionIdx = 0;
-        int lastHyphenIdx = -3;
-
-        RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
-        RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").position(currentTextPosition.getPosition()).build();
-
-        for (TextPositionSequence word : sequences) {
-            for (int i = 0; i < word.getTextPositions().size(); ++i) {
-
-                currentTextPosition = word.getTextPositions().get(i);
-
-                if (isLineBreak(currentTextPosition, previousTextPosition)) {
-
-                    if (stringIdx - lastHyphenIdx < 3) {
-                        sb.delete(lastHyphenIdx, sb.length());
-                        stringIdxToPositionIdx = stringIdxToPositionIdx.subList(0, lastHyphenIdx);
-                        stringIdx = lastHyphenIdx;
-                        lastHyphenIdx = -3;
-                    }
-                    lineBreaksStringIdx.add(stringIdx);
-                }
-                if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) {
-
-                    if (isHyphen(currentTextPosition.getUnicode())) {
-                        lastHyphenIdx = stringIdx;
-                    }
-                    sb.append(currentTextPosition.getUnicode());
-                    stringIdxToPositionIdx.add(positionIdx);
-                    ++stringIdx;
-                }
-
-                previousTextPosition = currentTextPosition;
-
-                ++positionIdx;
-            }
-
-            previousTextPosition = RedTextPosition.builder().unicode(" ").position(previousTextPosition.getPosition()).build();
-            sb.append(previousTextPosition.getUnicode());
-            stringIdxToPositionIdx.add(positionIdx);
-            ++stringIdx;
-        }
-
-        assert sb.length() == stringIdxToPositionIdx.size();
-
-        List<Rectangle2D> positions = sequences.stream()
-                .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence)))
-                .toList();
-
-        return SearchTextWithTextPositionModel.builder()
-                .searchText(sb.toString())
-                .lineBreaks(lineBreaksStringIdx)
-                .stringCoordsToPositionCoords(stringIdxToPositionIdx)
-                .positions(positions)
-                .build();
-    }
-
-
-    private static boolean isLineBreak(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition) {
-
-        return Objects.equals(currentTextPosition.getUnicode(), "\n") || isDeltaYLargerThanTextHeight(currentTextPosition, previousTextPosition);
-    }
-
-
-    private static boolean isDeltaYLargerThanTextHeight(RedTextPosition currentPosition, RedTextPosition previousPosition) {
-
-        if (previousPosition == null) {
-            return false;
-        }
-
-        float deltaY = Math.abs(currentPosition.getYDirAdj() - previousPosition.getYDirAdj());
-        return deltaY >= currentPosition.getHeightDir();
-    }
-
-
-    private static boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) {
-
-        return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " ");
-    }
-
-
-    private static boolean isHyphen(String unicodeCharacter) {
-
-        return Objects.equals(unicodeCharacter, "-") || //
-                Objects.equals(unicodeCharacter, "~") || //
-                Objects.equals(unicodeCharacter, "‐") || //
-                Objects.equals(unicodeCharacter, "‒") || //
-                Objects.equals(unicodeCharacter, "⁻") || //
-                Objects.equals(unicodeCharacter, "−") || //
-                Objects.equals(unicodeCharacter, "﹣") || //
-                Objects.equals(unicodeCharacter, "゠") || //
-                Objects.equals(unicodeCharacter, "⁓") || //
-                Objects.equals(unicodeCharacter, "‑") || //
-                Objects.equals(unicodeCharacter, "\u00AD");
-    }
-
-
-    private static Rectangle2D mapRedTextPositionToInitialUserSpace(RedTextPosition textPosition, TextPositionSequence sequence) {
-
-        float textHeight = sequence.getTextHeight() + HEIGHT_PADDING;
-        Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(),
-                textPosition.getYDirAdj() - textHeight,
-                textPosition.getWidthDirAdj(),
-                textHeight + HEIGHT_PADDING);
-
-        AffineTransform transform = new AffineTransform();
-
-        if (sequence.getDir() == TextDirection.ZERO || sequence.getDir() == TextDirection.HALF_CIRCLE) {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageWidth() / 2f, sequence.getPageHeight() / 2f);
-            transform.translate(0f, sequence.getPageHeight());
-        } else if (sequence.getDir() == TextDirection.QUARTER_CIRCLE) {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageWidth() / 2f, sequence.getPageWidth() / 2f);
-            transform.translate(0f, sequence.getPageWidth());
-        } else {
-            transform.rotate(sequence.getDir().getRadians(), sequence.getPageHeight() / 2f, sequence.getPageHeight() / 2f);
-            transform.translate(0f, sequence.getPageWidth());
-        }
-        transform.scale(1., -1.);
-
-        return transform.createTransformedShape(rectangle2D).getBounds2D();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/SearchTextWithTextPositionModel.java

@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.awt.geom.Rectangle2D;
-import java.util.List;
-
-import lombok.AccessLevel;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Builder
-@Getter
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class SearchTextWithTextPositionModel {
-
-    String searchText;
-    List<Integer> lineBreaks;
-    List<Integer> stringCoordsToPositionCoords;
-    List<Rectangle2D> positions;
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/TextBlockFactory.java

@@ -1,79 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.PageNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.internal.api.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-
-public class TextBlockFactory {
-
-    AtomicInteger stringOffset;
-    AtomicLong textBlockIdx;
-
-
-    public TextBlockFactory() {
-
-        stringOffset = new AtomicInteger();
-        textBlockIdx = new AtomicLong();
-    }
-
-
-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) {
-
-        Integer numberOnPage = context.pages().get(page).getAndIncrement();
-        return buildAtomicTextBlock(sequences, parent, context, numberOnPage, page);
-    }
-
-
-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences,
-                                                SemanticNode parent,
-                                                DocumentGraphFactory.Context context,
-                                                Integer numberOnPage,
-                                                PageNode page) {
-
-        SearchTextWithTextPositionModel searchTextWithTextPositionModel = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
-        int offset = stringOffset.getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
-
-        return AtomicTextBlock.builder()
-                .id(textBlockIdx.getAndIncrement())
-                .parent(parent)
-                .searchText(searchTextWithTextPositionModel.getSearchText())
-                .numberOnPage(numberOnPage)
-                .page(page)
-                .lineBreaks(searchTextWithTextPositionModel.getLineBreaks())
-                .positions(searchTextWithTextPositionModel.getPositions())
-                .stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords())
-                .boundary(new Boundary(offset, offset + searchTextWithTextPositionModel.getSearchText().length()))
-                .build();
-    }
-
-
-    public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, PageNode page) {
-
-        return emptyTextBlock(parent, context.pages().get(page).getAndIncrement(), page);
-    }
-
-
-    public AtomicTextBlock emptyTextBlock(SemanticNode parent, Integer numberOnPage, PageNode page) {
-
-        return AtomicTextBlock.builder()
-                .id(textBlockIdx.getAndIncrement())
-                .boundary(new Boundary(stringOffset.get(), stringOffset.get()))
-                .searchText("")
-                .lineBreaks(Collections.emptyList())
-                .page(page)
-                .numberOnPage(numberOnPage)
-                .stringIdxToPositionIdx(Collections.emptyList())
-                .positions(Collections.emptyList())
-                .parent(parent)
-                .build();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/factory/TextPositionOperations.java

@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.factory;
-
-import java.util.Comparator;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.ClassificationTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.TextPositionSequence;
-
-public class TextPositionOperations {
-
-    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<ClassificationTextBlock> textBlocks) {
-
-        return textBlocks.stream()//
-                .flatMap(tb -> tb.getSequences().stream())//
-                .sorted(Comparator.comparingDouble(TextPositionSequence::getMaxYDirAdj)//
-                        .thenComparing(TextPositionSequence::getMaxXDirAdj))//
-                .toList();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/AbstractPageBlock.java

@@ -0,0 +1,80 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public abstract class AbstractPageBlock {
+
+    @JsonIgnore
+    protected float minX;
+    @JsonIgnore
+    protected float maxX;
+    @JsonIgnore
+    protected float minY;
+    @JsonIgnore
+    protected float maxY;
+    @JsonIgnore
+    protected PageBlockType classification;
+    @JsonIgnore
+    protected int page;
+
+    @JsonIgnore
+    private Orientation orientation = Orientation.NONE;
+
+
+    public abstract String getText();
+
+
+    public boolean isHeadline() {
+
+        return this instanceof TextPageBlock && this.getClassification() != null && this.getClassification().isHeadline();
+    }
+
+
+    public boolean containsBlock(TextPageBlock other) {
+
+        return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY();
+    }
+
+
+    public boolean contains(AbstractPageBlock other) {
+
+        return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
+    }
+
+
+    public boolean contains(Rectangle other) {
+
+        return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft()
+                .getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
+    }
+
+
+    @JsonIgnore
+    public float getHeight() {
+
+        return maxY - minY;
+    }
+
+
+    @JsonIgnore
+    public float getWidth() {
+
+        return maxX - minX;
+    }
+
+
+    public boolean intersectsY(AbstractPageBlock atc) {
+
+        return this.minY <= atc.getMaxY() && this.maxY >= atc.getMinY();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationDocument.java

@@ -1,10 +1,11 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.ArrayList;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.StringFrequencyCounter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.UnclassifiedText;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText;
 
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -24,4 +25,7 @@ public class ClassificationDocument {
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
     private boolean headlines;
 
+    private SectionGrid sectionGrid = new SectionGrid();
+    private long rulesVersion;
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationFooter.java

@@ -0,0 +1,16 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class ClassificationFooter {
+
+    private List<TextPageBlock> textBlocks;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationHeader.java

@@ -0,0 +1,16 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class ClassificationHeader {
+
+    private List<TextPageBlock> textBlocks;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java

@@ -1,11 +1,12 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.ArrayList;
 import java.util.List;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.dto.text.StringFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 
 import lombok.Data;
 import lombok.NonNull;
@@ -16,7 +17,7 @@ import lombok.RequiredArgsConstructor;
 public class ClassificationPage {
 
     @NonNull
-    private List<AbstractTextContainer> textBlocks;
+    private List<AbstractPageBlock> textBlocks;
 
     private List<ClassifiedImage> images = new ArrayList<>();
 
@@ -35,4 +36,6 @@ public class ClassificationPage {
     private float pageWidth;
     private float pageHeight;
 
+    CleanRulings cleanRulings;
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationSection.java

@@ -0,0 +1,32 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class ClassificationSection {
+
+    private List<AbstractPageBlock> pageBlocks = new ArrayList<>();
+    private List<ClassifiedImage> images = new ArrayList<>();
+    private String headline;
+
+
+    public List<TablePageBlock> getTables() {
+
+        List<TablePageBlock> tables = new ArrayList<>();
+        pageBlocks.forEach(block -> {
+            if (block instanceof TablePageBlock) {
+                tables.add((TablePageBlock) block);
+            }
+        });
+        return tables;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/FloatFrequencyCounter.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.dto;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -9,9 +9,9 @@ import java.util.stream.Collectors;
 
 import lombok.Getter;
 
+@Getter
 public class FloatFrequencyCounter {
 
-    @Getter
     Map<Float, Integer> countPerValue = new HashMap<>();
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/GapInformation.java

@@ -0,0 +1,27 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class GapInformation {
+
+    List<List<Rectangle2D>> xGaps;
+    List<List<Rectangle2D>> yGaps;
+
+
+    public GapInformation() {
+
+        xGaps = new LinkedList<>();
+        yGaps = new LinkedList<>();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/LineInformation.java

@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class LineInformation {
+
+    List<Rectangle2D> lineBBox;
+    List<List<TextPositionSequence>> sequencesByLines;
+    List<List<Rectangle2D>> bBoxWithGapsByLines;
+    List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/Orientation.java

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+public enum Orientation {
+
+    NONE,
+    LEFT,
+    RIGHT
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageBlockType.java

@@ -0,0 +1,38 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+public enum PageBlockType {
+    H1,
+    H2,
+    H3,
+    H4,
+    H5,
+    H6,
+    HEADER,
+    FOOTER,
+    TITLE,
+    PARAGRAPH,
+    PARAGRAPH_BOLD,
+    PARAGRAPH_ITALIC,
+    PARAGRAPH_UNKNOWN,
+    OTHER,
+    TABLE;
+
+
+    public static PageBlockType getHeadlineType(int i) {
+
+        return switch (i) {
+            case 1 -> PageBlockType.H1;
+            case 2 -> PageBlockType.H2;
+            case 3 -> PageBlockType.H3;
+            case 4 -> PageBlockType.H4;
+            case 5 -> PageBlockType.H5;
+            default -> PageBlockType.H6;
+        };
+    }
+
+
+    public boolean isHeadline() {
+
+        return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4) || this.equals(H5) || this.equals(H6);
+    }
+}