Native Build

Update dependency com.iqser.red.commons:storage-commons to v2.36.0 (main)

See merge request fforesight/layout-parser!50

.gitignore

@@ -18,6 +18,7 @@ target/
 .settings
 .springBeans
 .sts4-cache
+.gradle
 
 ### IntelliJ IDEA ###
 .idea
@@ -37,3 +38,7 @@ build/
 
 ### VS Code ###
 .vscode/
+gradlew.bat
+gradlew
+gradle.properties
+gradle/

.gitlab-ci.yml

@@ -1,4 +1,21 @@
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'
-    file: 'ci-templates/maven_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG

.mvn/wrapper/maven-wrapper.properties

@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.7/apache-maven-3.8.7-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar

buildSrc/build.gradle.kts

@@ -0,0 +1,7 @@
+plugins {
+    `kotlin-dsl`
+}
+
+repositories {
+    gradlePluginPortal()
+}

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -0,0 +1,75 @@
+plugins {
+    `java-library`
+    `maven-publish`
+    pmd
+    checkstyle
+    jacoco
+}
+
+group = "com.knecon.fforesight"
+
+java.sourceCompatibility = JavaVersion.VERSION_17
+java.targetCompatibility = JavaVersion.VERSION_17
+
+tasks.pmdMain {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
+}
+
+tasks.pmdTest {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/test_pmd.xml")
+}
+
+tasks.named<Test>("test") {
+    useJUnitPlatform()
+    reports {
+        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
+    }
+}
+
+tasks.test {
+    finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
+}
+
+tasks.jacocoTestReport {
+    dependsOn(tasks.test) // tests are required to run before generating the report
+    reports {
+        xml.required.set(true)
+        csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
+    }
+}
+
+allprojects {
+    publishing {
+        publications {
+            create<MavenPublication>(name) {
+                from(components["java"])
+            }
+        }
+        repositories {
+            maven {
+                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+                credentials {
+                    username = providers.gradleProperty("mavenUser").getOrNull();
+                    password = providers.gradleProperty("mavenPassword").getOrNull();
+                }
+            }
+        }
+    }
+}
+
+java {
+    withJavadocJar()
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven {
+        url = uri("https://nexus.knecon.com/repository/gindev/")
+        credentials {
+            username = providers.gradleProperty("mavenUser").getOrNull();
+            password = providers.gradleProperty("mavenPassword").getOrNull();
+        }
+    }
+}

config/checkstyle/checkstyle.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <property
+            name="severity"
+            value="error"/>
+    <module name="TreeWalker">
+        <module name="SuppressWarningsHolder"/>
+        <module name="MissingDeprecated"/>
+        <module name="MissingOverride"/>
+        <module name="AnnotationLocation"/>
+        <module name="JavadocStyle"/>
+        <module name="NonEmptyAtclauseDescription"/>
+        <module name="IllegalImport"/>
+        <module name="RedundantImport"/>
+        <module name="RedundantModifier"/>
+        <module name="EmptyBlock"/>
+        <module name="DefaultComesLast"/>
+        <module name="EmptyStatement"/>
+        <module name="EqualsHashCode"/>
+        <module name="ExplicitInitialization"/>
+        <module name="IllegalInstantiation"/>
+        <module name="ModifiedControlVariable"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="PackageDeclaration"/>
+        <module name="ParameterAssignment"/>
+        <module name="SimplifyBooleanExpression"/>
+        <module name="SimplifyBooleanReturn"/>
+        <module name="StringLiteralEquality"/>
+        <module name="OneStatementPerLine"/>
+        <module name="FinalClass"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="UpperEll"/>
+        <module name="OuterTypeFilename"/>
+    </module>
+    <module name="FileTabCharacter"/>
+    <module name="SuppressWarningsFilter"/>
+</module>

config/pmd/pmd.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon ruleset checks the code for bad stuff
+    </description>
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+    </rule>
+
+</ruleset>

config/pmd/test_pmd.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon test ruleset checks the code for bad stuff
+    </description>
+
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="AvoidFieldNameMatchingTypeName"/>
+        <exclude name="TestClassWithoutTestCases"/>
+    </rule>
+
+</ruleset>

gradle.properties.kts

@@ -0,0 +1 @@
+version = 0.1-SNAPSHOT

layoutparser-service-image/pom.xml

@@ -1,99 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>platform-docker-dependency</artifactId>
-        <version>0.1.0</version>
-        <relativePath/>
-    </parent>
-
-    <groupId>com.knecon.fforesight</groupId>
-    <artifactId>layoutparser-service-image</artifactId>
-    <version>0.1-SNAPSHOT</version>
-    <packaging>pom</packaging>
-
-    <properties>
-        <service.server>layoutparser-service-server</service.server>
-        <platform.jar>${service.server}.jar</platform.jar>
-        <docker.skip.push>false</docker.skip.push>
-        <docker.image.prefix>ff</docker.image.prefix>
-        <docker.image.name>${docker.image.prefix}/${service.server}</docker.image.name>
-    </properties>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-dependency-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-resources-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>io.fabric8</groupId>
-                <artifactId>docker-maven-plugin</artifactId>
-            </plugin>
-        </plugins>
-
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-dependency-plugin</artifactId>
-                    <executions>
-                        <execution>
-                            <id>download-platform-jar</id>
-                            <phase>prepare-package</phase>
-                            <goals>
-                                <goal>copy</goal>
-                            </goals>
-                            <configuration>
-                                <artifactItems>
-                                    <dependency>
-                                        <groupId>${project.groupId}</groupId>
-                                        <artifactId>${service.server}</artifactId>
-                                        <version>${project.version}</version>
-                                        <type>jar</type>
-                                        <overWrite>true</overWrite>
-                                        <destFileName>${platform.jar}</destFileName>
-                                    </dependency>
-                                </artifactItems>
-                                <outputDirectory>${docker.build.directory}</outputDirectory>
-                            </configuration>
-                        </execution>
-                    </executions>
-                </plugin>
-                <plugin>
-                    <groupId>io.fabric8</groupId>
-                    <artifactId>docker-maven-plugin</artifactId>
-                    <configuration>
-                        <images>
-                            <image>
-                                <name>${docker.image.name}</name>
-                                <build>
-                                    <dockerFileDir>${docker.build.directory}</dockerFileDir>
-                                    <args>
-                                        <PLATFORM_JAR>${platform.jar}</PLATFORM_JAR>
-                                    </args>
-                                    <tags>
-                                        <tag>${docker.image.version}</tag>
-                                        <tag>latest</tag>
-                                    </tags>
-                                </build>
-                            </image>
-                        </images>
-                    </configuration>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
-</project>

layoutparser-service-image/src/main/docker/Dockerfile

@@ -1,9 +0,0 @@
-FROM red/base-image:2.0.2
-
-ARG PLATFORM_JAR
-
-ENV PLATFORM_JAR ${PLATFORM_JAR}
-
-ENV USES_ELASTICSEARCH false
-
-COPY ["${PLATFORM_JAR}", "/"]

layoutparser-service/layoutparser-service-internal-api/build.gradle.kts

@@ -0,0 +1,6 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.2.2"
+}
+
+description = "layoutparser-service-internal-api"

layoutparser-service/layoutparser-service-internal-api/pom.xml

@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>0.1-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>layoutparser-service-internal-api</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>${guava.version}</version>
-        </dependency>
-    </dependencies>
-
-</project>

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentData.java

@@ -17,5 +17,4 @@ public class DocumentData {
     DocumentPositionData[] documentPositions;
     DocumentStructure documentStructure;
 
-
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPage.java

@@ -4,12 +4,14 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@FieldDefaults(level = AccessLevel.PRIVATE)
 public class DocumentPage {
 
     int number;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPositionData.java

@@ -4,12 +4,14 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@FieldDefaults(level = AccessLevel.PRIVATE)
 public class DocumentPositionData {
 
     Long id;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.awt.geom.Rectangle2D;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -8,20 +10,53 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
-@AllArgsConstructor
 @NoArgsConstructor
+@AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class DocumentStructure {
 
     EntryData root;
 
 
+    public static class TableProperties {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    public static class ImageProperties {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+    }
+
+    public static class TableCellProperties {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
     public EntryData get(List<Integer> tocId) {
 
         if (tocId.isEmpty()) {
@@ -53,10 +88,11 @@ public class DocumentStructure {
     }
 
 
+    @Data
     @Builder
-    @Getter
+    @NoArgsConstructor
     @AllArgsConstructor
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    @FieldDefaults(level = AccessLevel.PRIVATE)
     public static class EntryData {
 
         NodeType type;
@@ -66,6 +102,7 @@ public class DocumentStructure {
         Map<String, String> properties;
         List<EntryData> children;
 
+
         @Override
         public String toString() {
 

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentTextData.java

@@ -6,12 +6,14 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@FieldDefaults(level = AccessLevel.PRIVATE)
 public class DocumentTextData {
 
     Long id;
@@ -22,6 +24,4 @@ public class DocumentTextData {
     int end;
     int[] lineBreaks;
 
-
-
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedSectionText.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.text;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import java.util.ArrayList;
 import java.util.List;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/TableData.java

@@ -12,4 +12,5 @@ public class TableData {
     List<RowData> rowData;
     Integer numberOfCols;
     Integer numberOfRows;
+
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -4,20 +4,22 @@ import java.util.Map;
 import java.util.Optional;
 
 import lombok.Builder;
-
+import lombok.NonNull;
 
 @Builder
 public record LayoutParsingRequest(
-        LayoutParsingType layoutParsingType,
+        @NonNull LayoutParsingType layoutParsingType,
         Map<String, String> identifier,
-        String originFileStorageId,
+        @NonNull String originFileStorageId,
         Optional<String> tablesFileStorageId,
         Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
+        @NonNull String structureFileStorageId,
         String researchDocumentStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId,
-        String sectionGridStorageId) {
+        @NonNull String textBlockFileStorageId,
+        @NonNull String positionBlockFileStorageId,
+        @NonNull String pageFileStorageId,
+        @NonNull String simplifiedTextStorageId,
+        @NonNull String viewerDocumentStorageId,
+        @NonNull String sectionGridStorageId) {
 
 }

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -0,0 +1,26 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.2.2"
+}
+
+description = "layoutparser-service-processor"
+
+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0-RC1"
+
+dependencies {
+    implementation(project(":layoutparser-service-internal-api"))
+
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.138.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
+    }
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("com.iqser.red.commons:storage-commons:2.36.0")
+
+    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.1.2")
+}

layoutparser-service/layoutparser-service-processor/pom.xml

@@ -1,71 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>0.1-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>layoutparser-service-processor</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-shared-api-v1</artifactId>
-            <version>2.36.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>tenant-commons</artifactId>
-            <version>${tennat-commons.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>layoutparser-service-internal-api</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>storage-commons</artifactId>
-            <version>${storage-commons.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox-tools</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-afterburner</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-jsr310</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-web</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-amqp</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>RELEASE</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-</project>

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -2,31 +2,57 @@ package com.knecon.fforesight.service.layoutparser.processor;
 
 import static java.lang.String.format;
 
+import java.awt.geom.Rectangle2D;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.mapper.taas.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
+import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
 
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -37,63 +63,140 @@ public class LayoutParsingPipeline {
     private final ImageServiceResponseAdapter imageServiceResponseAdapter;
     private final CvTableParsingAdapter cvTableParsingAdapter;
     private final LayoutParsingStorageService layoutParsingStorageService;
-    private final PdfParsingService pdfParsingService;
     private final SectionsBuilderService sectionsBuilderService;
     private final SectionGridCreatorService sectionGridCreatorService;
     private final TaasClassificationService taasClassificationService;
     private final RedactManagerClassificationService redactManagerClassificationService;
     private final DocuMineClassificationService docuMineClassificationService;
+    private final SimplifiedSectionTextService simplifiedSectionTextService;
+    private final BodyTextFrameService bodyTextFrameService;
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;
+    private final ViewerDocumentService viewerDocumentService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
 
         long start = System.currentTimeMillis();
-        PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
 
-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
+        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
+            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            }
+
+            TableServiceResponse tableServiceResponse = new TableServiceResponse();
+            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+            }
+
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+
+            int numberOfPages = originDocument.getNumberOfPages();
+
+            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
+            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+            try (var out = new ByteArrayOutputStream()) {
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out);
+                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
+            }
+
+            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+            }
+
+            return LayoutParsingFinishedEvent.builder()
+                    .identifier(layoutParsingRequest.identifier())
+                    .numberOfPages(numberOfPages)
+                    .duration(System.currentTimeMillis() - start)
+                    .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
+                            layoutParsingRequest.structureFileStorageId(),
+                            layoutParsingRequest.textBlockFileStorageId(),
+                            layoutParsingRequest.positionBlockFileStorageId(),
+                            layoutParsingRequest.pageFileStorageId()))
+                    .build();
         }
-
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
-        }
-
-        Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
-        int numberOfPages = originDocument.getNumberOfPages();
-        originDocument.close();
-
-        layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-
-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
-            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
-            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
-        }
-
-        return LayoutParsingFinishedEvent.builder()
-                .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(numberOfPages)
-                .duration(System.currentTimeMillis() - start)
-                .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId()))
-                .build();
     }
 
 
-    public Document parseLayout(LayoutParsingType layoutParsingType,
-                                PDDocument originDocument,
-                                ImageServiceResponse imageServiceResponse,
-                                TableServiceResponse tableServiceResponse) {
+    @SneakyThrows
+    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
+                                              PDDocument originDocument,
+                                              ImageServiceResponse imageServiceResponse,
+                                              TableServiceResponse tableServiceResponse) {
 
-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
-                originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
+        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
+        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
+
+        ClassificationDocument classificationDocument = new ClassificationDocument();
+        List<ClassificationPage> classificationPages = new ArrayList<>();
+
+        originDocument.setAllSecurityToBeRemoved(true);
+        long pageCount = originDocument.getNumberOfPages();
+
+        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+            classificationDocument.setPages(classificationPages);
+            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+            PDPage pdPage = originDocument.getPage(pageNumber - 1);
+            stripper.setPageNumber(pageNumber);
+            stripper.setStartPage(pageNumber);
+            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE)) {
+                stripper.setSortByPosition(true);
+            }
+            stripper.getText(originDocument);
+
+            PDRectangle pdr = pdPage.getMediaBox();
+
+            int rotation = pdPage.getRotation();
+            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+            PDRectangle cropbox = pdPage.getCropBox();
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                    stripper.getRulings(),
+                    stripper.getMinCharWidth(),
+                    stripper.getMaxCharHeight());
+
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER ->
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS ->
+                        taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE ->
+                        docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            };
+            classificationPage.setCleanRulings(cleanRulings);
+            classificationPage.setRotation(rotation);
+            classificationPage.setLandscape(isLandscape);
+            classificationPage.setPageNumber(pageNumber);
+            classificationPage.setPageWidth(cropbox.getWidth());
+            classificationPage.setPageHeight(cropbox.getHeight());
+
+            // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
+            classificationPage.setMarkedContentBboxPerType(convertMarkedContents(stripper.getMarkedContents()));
+
+            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+                classificationPage.setImages(pdfImages.get(pageNumber));
+                imageServiceResponseAdapter.findOcr(classificationPage);
+            }
+
+            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            buildPageStatistics(classificationPage);
+            increaseDocumentStatistics(classificationPage, classificationDocument);
+
+            classificationPages.add(classificationPage);
+        }
+
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
 
         switch (layoutParsingType) {
             case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
@@ -102,40 +205,47 @@ public class LayoutParsingPipeline {
         }
 
         sectionsBuilderService.buildSections(classificationDocument);
+        sectionsBuilderService.addImagesToSections(classificationDocument);
 
-        return DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+        return classificationDocument;
     }
 
 
-    public Document parseLayoutWithTimer(LayoutParsingType layoutParsingType,
-                                         PDDocument originDocument,
-                                         ImageServiceResponse imageServiceResponse,
-                                         TableServiceResponse tableServiceResponse) {
+    private Map<String, Rectangle2D> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+        Map<String, Rectangle2D> markedContentBboxes = new HashMap<>();
+        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
+        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
+        return markedContentBboxes;
+    }
 
-        long start = System.currentTimeMillis();
+    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
 
-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType, originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
-
-        System.out.printf("parsed %d ms", System.currentTimeMillis() - start);
-
-        start = System.currentTimeMillis();
-        switch (layoutParsingType) {
-            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
-            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
+        if (!classificationPage.isLandscape()) {
+            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
         }
-        System.out.printf(", classified %d ms", System.currentTimeMillis() - start);
+        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
+        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
+        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
+    }
 
-        start = System.currentTimeMillis();
-        sectionsBuilderService.buildSections(classificationDocument);
-        System.out.printf(", sections built %d ms", System.currentTimeMillis() - start);
 
-        start = System.currentTimeMillis();
-        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-        System.out.printf(", graph constructed %d ms", System.currentTimeMillis() - start);
-        return document;
+    private void buildPageStatistics(ClassificationPage classificationPage) {
+
+        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
+        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
+                    continue;
+                }
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
+                    classificationPage.getFontCounter().add(word.getFont());
+                    classificationPage.getFontSizeCounter().add(word.getFontSize());
+                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
+                }
+            }
+        }
+
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -1,10 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
@@ -15,18 +20,16 @@ import org.springframework.stereotype.Service;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -40,10 +43,11 @@ public class LayoutParsingStorageService {
 
     public PDDocument getOriginFile(String storageId) throws IOException {
 
-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var originDocumentInputStream = getObject(storageId)) {
             File tempFile = createTempFile("document", ".pdf");
             try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
                 IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
             }
             return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
         }
@@ -52,19 +56,22 @@ public class LayoutParsingStorageService {
 
     public ImageServiceResponse getImagesFile(String storageId) throws IOException {
 
-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (InputStream inputStream = getObject(storageId)) {
 
-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
         }
     }
 
 
     public TableServiceResponse getTablesFile(String storageId) throws IOException {
 
-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+        try (var tableClassificationStream = getObject(storageId)) {
 
+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
         }
     }
 
@@ -90,26 +97,6 @@ public class LayoutParsingStorageService {
     }
 
 
-    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
-
-        DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class);
-        DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.textBlockFileStorageId(),
-                DocumentTextData[].class);
-        DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.positionBlockFileStorageId(),
-                DocumentPositionData[].class);
-        DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class);
-
-        return DocumentData.builder()
-                .documentStructure(tableOfContentsData)
-                .documentPositions(atomicPositionBlockData)
-                .documentTextData(documentTextDataBlockData)
-                .documentPages(documentPageData)
-                .build();
-    }
-
-
     private File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {
 
         File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();
@@ -134,4 +121,30 @@ public class LayoutParsingStorageService {
         }
     }
 
+
+    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
+    }
+
+
+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
+
+    @SneakyThrows
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {
+
+        try (var in = new ByteArrayInputStream(out.toByteArray())) {
+
+            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
+        }
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Classification.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import lombok.Data;
-
-@Data
-public class Classification {
-
-    private Map<String, Float> probabilities = new HashMap<>();
-    private String label;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/FilterGeometry.java

@@ -1,11 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class FilterGeometry {
-
-    private ImageSize imageSize;
-    private ImageFormat imageFormat;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Filters.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Filters {
-
-    private FilterGeometry geometry;
-    private Probability probability;
-    private boolean allPassed;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Geometry.java

@@ -1,11 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Geometry {
-
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageFormat.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageFormat {
-
-    private float quotient;
-    private boolean tooTall;
-    private boolean tooWide;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageMetadata.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageMetadata {
-
-    private Classification classification;
-    private Position position;
-    private Geometry geometry;
-    private Filters filters;
-    private boolean alpha;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageSize.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageSize {
-
-    private float quotient;
-    private boolean tooLarge;
-    private boolean tooSmall;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Position.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Position {
-
-    private float x1;
-    private float x2;
-    private float y1;
-    private float y2;
-    private int pageNumber;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Probability.java

@@ -1,10 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Probability {
-
-    private boolean unconfident;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/PageInfo.java

@@ -1,13 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import lombok.Data;
-
-@Data
-public class PageInfo {
-
-    private int number;
-    private int rotation;
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/TableData.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import lombok.Data;
-
-@Data
-public class TableData {
-
-    private PageInfo pageInfo;
-    private List<TableCells> tableCells = new ArrayList<>();
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/GenericSemanticNode.java

@@ -1,5 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-public interface GenericSemanticNode extends SemanticNode {
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/mapper/redaction/PropertiesMapper.java

@@ -1,112 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-
-public class PropertiesMapper {
-
-    public static final String TRANSPARENT = "transparent";
-    public static final String IMAGE_TYPE = "imageType";
-    public static final String POSITION = "position";
-    public static final String ROW = "row";
-    public static final String COL = "col";
-    public static final String HEADER = "header";
-    public static final String B_BOX = "bBox";
-    public static final String NUMBER_OF_ROWS = "numberOfRows";
-    public static final String NUMBER_OF_COLS = "numberOfCols";
-
-
-    public static Map<String, String> buildImageProperties(Image image) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put(IMAGE_TYPE, image.getImageType().toString());
-        properties.put(TRANSPARENT, String.valueOf(image.isTransparent()));
-        properties.put(POSITION, RectangleTransformations.toString(image.getPosition()));
-        return properties;
-    }
-
-
-    public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put(ROW, String.valueOf(tableCell.getRow()));
-        properties.put(COL, String.valueOf(tableCell.getCol()));
-        properties.put(HEADER, String.valueOf(tableCell.isHeader()));
-
-        if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
-            throw new IllegalArgumentException("TableCell can only occur on a single page!");
-        }
-        String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
-        properties.put(B_BOX, bBoxString);
-
-        return properties;
-    }
-
-
-    public static Map<String, String> buildTableProperties(Table table) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put(NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
-        properties.put(NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
-        return properties;
-    }
-
-
-    public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
-
-        builder.imageType(parseImageType(properties.get(IMAGE_TYPE)));
-        builder.transparent(Boolean.parseBoolean(properties.get(TRANSPARENT)));
-        builder.position(parseRectangle2D(properties.get(POSITION)));
-    }
-
-
-    public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
-
-        builder.row(Integer.parseInt(properties.get(ROW)));
-        builder.col(Integer.parseInt(properties.get(COL)));
-        builder.header(Boolean.parseBoolean(properties.get(HEADER)));
-        builder.bBox(parseRectangle2D(properties.get(B_BOX)));
-    }
-
-
-    public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
-
-        builder.numberOfRows(Integer.parseInt(properties.get(NUMBER_OF_ROWS)));
-        builder.numberOfCols(Integer.parseInt(properties.get(NUMBER_OF_COLS)));
-    }
-
-
-    private static ImageType parseImageType(String imageType) {
-
-        return switch (imageType) {
-            case "LOGO" -> ImageType.LOGO;
-            case "FORMULA" -> ImageType.FORMULA;
-            case "SIGNATURE" -> ImageType.SIGNATURE;
-            case "OCR" -> ImageType.OCR;
-            default -> ImageType.OTHER;
-        };
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return String.format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java

@@ -1,15 +1,20 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;
 
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 
 @Data
 @RequiredArgsConstructor
@@ -35,4 +40,8 @@ public class ClassificationPage {
     private float pageWidth;
     private float pageHeight;
 
+    CleanRulings cleanRulings;
+
+    private Map<String, Rectangle2D> markedContentBboxPerType = new HashMap<>();
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageContents.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;
 
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 
 import lombok.AllArgsConstructor;
@@ -17,5 +18,5 @@ public class PageContents {
     List<TextPositionSequence> sortedTextPositionSequences;
     Rectangle2D cropBox;
     Rectangle2D mediaBox;
-
+    List<Ruling> rulings;
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/Boundary.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph;
 
 import static java.lang.String.format;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/DocumentTree.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph;
 
 import static java.lang.String.format;
 
@@ -8,13 +8,13 @@ import java.util.List;
 import java.util.stream.Stream;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/entity/EntityType.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
 
 public enum EntityType {
     ENTITY,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/entity/RedactionEntity.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
@@ -11,9 +11,9 @@ import java.util.Map;
 import java.util.Set;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 
 import lombok.AccessLevel;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/entity/RedactionPosition.java

@@ -1,9 +1,9 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
 
 import java.awt.geom.Rectangle2D;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -12,10 +12,10 @@ import java.util.stream.Stream;
 
 import com.amazonaws.services.kms.model.NotFoundException;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/GenericSemanticNode.java

@@ -0,0 +1,5 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+
+public interface GenericSemanticNode extends SemanticNode {
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -9,10 +9,10 @@ import java.util.Map;
 import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java

@@ -1,4 +1,6 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+
+import java.util.Locale;
 
 public enum ImageType {
     LOGO,
@@ -10,7 +12,7 @@ public enum ImageType {
 
     public static ImageType fromString(String imageType) {
 
-        return switch (imageType.toLowerCase()) {
+        return switch (imageType.toLowerCase(Locale.ROOT)) {
             case "logo" -> ImageType.LOGO;
             case "formula" -> ImageType.FORMULA;
             case "signature" -> ImageType.SIGNATURE;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Page.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -6,9 +6,9 @@ import java.util.List;
 import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java

@@ -1,14 +1,14 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import static java.lang.String.format;
 
@@ -6,18 +6,19 @@ import java.awt.geom.Rectangle2D;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.EntityType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.EntityType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 public interface SemanticNode {
@@ -285,7 +286,7 @@ public interface SemanticNode {
      */
     default boolean containsStringIgnoreCase(String string) {
 
-        return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
+        return getTextBlock().getSearchText().toLowerCase(Locale.ROOT).contains(string.toLowerCase(Locale.ROOT));
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java

@@ -1,19 +1,20 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import static java.lang.String.format;
 
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -67,7 +68,7 @@ public class Table implements SemanticNode {
      */
     public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
 
-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase();
+        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
         return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
@@ -8,10 +8,10 @@ import java.util.Map;
 import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/AtomicTextBlock.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
 import static java.lang.String.format;
 
@@ -13,9 +13,9 @@ import java.util.Map;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.AccessLevel;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/ConcatenatedTextBlock.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
 import static java.lang.String.format;
 
@@ -11,8 +11,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 
 import lombok.AccessLevel;
 import lombok.Data;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/TextBlock.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
 import static java.lang.String.format;
 
@@ -10,8 +10,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 
 public interface TextBlock extends CharSequence {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/TextBlockCollector.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
 import java.util.Set;
 import java.util.function.BiConsumer;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java

@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.image;
 
 import java.awt.geom.Rectangle2D;
 
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 
 import lombok.Data;
 import lombok.NonNull;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Ruling.java

@@ -29,6 +29,24 @@ public class Ruling extends Line2D.Float {
     }
 
 
+    public Ruling straightenVertical() {
+
+        double y1 = Math.min(getY1(), getY2());
+        double y2 = Math.max(getY1(), getY2());
+        double x = (getX1() + getX2()) / 2;
+        return new Ruling(new Point2D.Double(x, y1), new Point2D.Double(x, y2));
+    }
+
+
+    public Ruling straightenHorizontal() {
+
+        double x1 = Math.min(getX1(), getX2());
+        double x2 = Math.max(getX1(), getX2());
+        double y = (getY1() + getY2()) / 2;
+        return new Ruling(new Point2D.Double(x1, y), new Point2D.Double(x2, y));
+    }
+
+
     public static List<Ruling> cropRulingsToArea(List<Ruling> rulings, Rectangle2D area) {
 
         ArrayList<Ruling> rv = new ArrayList<>();
@@ -428,6 +446,16 @@ public class Ruling extends Line2D.Float {
     }
 
 
+    public boolean almostMatches(Ruling ruling) {
+
+        final float TOLERANCE = 1;
+        return Math.abs(ruling.getX1() - x1) < TOLERANCE &&//
+                Math.abs(ruling.getY1() - y1) < TOLERANCE &&//
+                Math.abs(ruling.getX2() - x2) < TOLERANCE &&//
+                Math.abs(ruling.getY2() - y2) < TOLERANCE;
+    }
+
+
     private enum SOType {
         VERTICAL,
         HRIGHT,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java

@@ -73,6 +73,7 @@ public class TextPageBlock extends AbstractPageBlock {
 
         return sequences.get(0).getPageWidth();
     }
+    
 
     public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -4,6 +4,7 @@ import java.awt.geom.AffineTransform;
 import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.stream.Collectors;
 
 import org.apache.pdfbox.text.TextPosition;
@@ -76,7 +77,7 @@ public class TextPositionSequence implements CharSequence {
 
         RedTextPosition textPosition = textPositionAt(index);
         String text = textPosition.getUnicode();
-        return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
+        return caseInSensitive ? text.toLowerCase(Locale.ROOT).charAt(0) : text.charAt(0);
     }
 
 
@@ -223,7 +224,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public String getFont() {
 
-        return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
+        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
     }
 
 
@@ -231,7 +232,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public String getFontStyle() {
 
-        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();
+        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);
 
         if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
             return "bold, italic";

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/visualization/ColoredLine.java

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Line2D;
+
+public record ColoredLine(Line2D line, Color color) {
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/visualization/ColoredRectangle.java

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
+
+public record ColoredRectangle(Rectangle2D rectangle2D, Color color) {
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/visualization/FilledRectangle.java

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
+
+public record FilledRectangle(Rectangle2D rectangle2D, Color color, float alpha) {
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/visualization/LayoutGrid.java

@@ -0,0 +1,27 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class LayoutGrid {
+
+    int numberOfPages;
+    List<VisualizationsOnPage> visualizationsPerPages;
+
+
+    public LayoutGrid(int numberOfPages) {
+
+        this.numberOfPages = numberOfPages;
+        this.visualizationsPerPages = new ArrayList<>(numberOfPages);
+        for (int i = 0; i < numberOfPages; i++) {
+            this.visualizationsPerPages.add(VisualizationsOnPage.builder().pageNumber(i).build());
+        }
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/visualization/PlacedText.java

@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.geom.Point2D;
+
+public record PlacedText(String text, Point2D lineStart) {
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/visualization/VisualizationsOnPage.java

@@ -0,0 +1,26 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@Builder
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class VisualizationsOnPage {
+
+    int pageNumber;
+    @Builder.Default
+    List<PlacedText> placedTexts = new LinkedList<>();
+    @Builder.Default
+    List<ColoredLine> coloredLines = new LinkedList<>();
+    @Builder.Default
+    List<ColoredRectangle> coloredRectangles = new LinkedList<>();
+    @Builder.Default
+    List<FilledRectangle> filledRectangles = new LinkedList<>();
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/CvTableParsingAdapter.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter;
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -8,8 +8,8 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/ImageServiceResponseAdapter.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter;
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
 
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
@@ -9,10 +9,10 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 
 import lombok.RequiredArgsConstructor;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Classification.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Classification {
+
+    private Map<String, Float> probabilities = new HashMap<>();
+    private String label;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/FilterGeometry.java

@@ -0,0 +1,17 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class FilterGeometry {
+
+    private ImageSize imageSize;
+    private ImageFormat imageFormat;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Filters.java

@@ -0,0 +1,18 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Filters {
+
+    private FilterGeometry geometry;
+    private Probability probability;
+    private boolean allPassed;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Geometry.java

@@ -0,0 +1,17 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Geometry {
+
+    private float width;
+    private float height;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageFormat.java

@@ -0,0 +1,18 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImageFormat {
+
+    private float quotient;
+    private boolean tooTall;
+    private boolean tooWide;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageMetadata.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImageMetadata {
+
+    private Classification classification;
+    private Position position;
+    private Geometry geometry;
+    private Filters filters;
+    private boolean alpha;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageServiceResponse.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -6,9 +6,15 @@ import java.util.List;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class ImageServiceResponse {
 
     private String dossierId;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/ImageSize.java

@@ -0,0 +1,18 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImageSize {
+
+    private float quotient;
+    private boolean tooLarge;
+    private boolean tooSmall;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Position.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Position {
+
+    private float x1;
+    private float x2;
+    private float y1;
+    private float y2;
+    private int pageNumber;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/image/Probability.java

@@ -0,0 +1,16 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Probability {
+
+    private boolean unconfident;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/PageInfo.java

@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class PageInfo {
+
+    private int number;
+    private int rotation;
+    private float width;
+    private float height;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/PdfTableCell.java

@@ -1,14 +1,14 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.RequiredArgsConstructor;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@RequiredArgsConstructor
 public class PdfTableCell {
 
     private float x0;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableCells.java

@@ -1,10 +1,14 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
 
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class TableCells {
 
     private float x0;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableData.java

@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableData {
+
+    private PageInfo pageInfo;
+    private List<TableCells> tableCells = new ArrayList<>();
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/TableServiceResponse.java

@@ -1,11 +1,17 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 public class TableServiceResponse {
 
     private String dossierId;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessagingConfiguration.java

@@ -1,40 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.queue;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
-
-import org.springframework.amqp.core.Queue;
-import org.springframework.amqp.core.QueueBuilder;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-import lombok.RequiredArgsConstructor;
-
-@Configuration
-@RequiredArgsConstructor
-public class MessagingConfiguration {
-
-    @Bean
-    public Queue layoutparsingRequestQueue() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingResponseQueue() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingDLQ() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/BodyTextFrameService.java

@@ -2,11 +2,14 @@ package com.knecon.fforesight.service.layoutparser.processor.services;
 
 import java.util.List;
 
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -17,7 +20,65 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 @Service
 public class BodyTextFrameService {
 
-    private static final float APPROXIMATE_HEADER_LINE_COUNT = 2.0f;
+
+    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {
+
+        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
+        Rectangle landscapeBodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), true, layoutParsingType);
+        for (ClassificationPage page : classificationDocument.getPages()) {
+            setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
+        }
+    }
+
+
+/*
+    private Rectangle calculateBodyTextFrameByRulings(List<ClassificationPage> pages) {
+
+        Map<ClassificationPage, List<Ruling>> potentialHeaderRulingsPerPage = new HashMap<>();
+        Map<ClassificationPage, List<Ruling>> potentialFooterRulingsPerPage = new HashMap<>();
+
+        for (var page : pages) {
+            potentialHeaderRulingsPerPage.put(page,
+                    page.getCleanRulings()
+                            .getHorizontal()
+                            .stream()
+                            .filter(ruling -> ruling.getY1() > page.getPageHeight() * 0.8)
+                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
+                            .toList());
+            potentialFooterRulingsPerPage.put(page,
+                    page.getCleanRulings()
+                            .getHorizontal()
+                            .stream()
+                            .filter(ruling -> ruling.getY1() < page.getPageHeight() * 0.2)
+                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
+                            .toList());
+        }
+
+        Optional<Ruling> headerRuling = potentialHeaderRulingsPerPage.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(ruling -> potentialHeaderRulingsPerPage.values()
+                        .stream()
+                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
+                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
+                .min(Comparator.comparingDouble(Ruling::getY1));
+
+        Optional<Ruling> footerRuling = potentialFooterRulingsPerPage.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(ruling -> potentialHeaderRulingsPerPage.values()
+                        .stream()
+                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
+                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
+                .max(Comparator.comparingDouble(Ruling::getY1));
+
+        double maxY = headerRuling.isPresent() ? headerRuling.get().y1 : pages.stream().mapToDouble(ClassificationPage::getPageHeight).max().orElse(Double.MAX_VALUE);
+        double minY = footerRuling.map(ruling -> ruling.y1).orElse(0F);
+        double maxX = pages.stream().mapToDouble(ClassificationPage::getPageWidth).max().orElse(Double.MAX_VALUE);
+
+        return new Rectangle(new Point((float) maxX, (float) maxY), (float) 0, (float) minY, -1);
+    }
+*/
 
 
     /**
@@ -33,7 +94,7 @@ public class BodyTextFrameService {
      * @param bodyTextFrame          frame that contains the main text on portrait pages
      * @param landscapeBodyTextFrame frame that contains the main text on landscape pages
      */
-    public void setBodyTextFrameAdjustedToPage(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
+    private void setBodyTextFrameAdjustedToPage(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
 
         Rectangle textFrame = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
 
@@ -68,7 +129,17 @@ public class BodyTextFrameService {
      * @param landscape               Calculate for landscape or portrait
      * @return Rectangle of the text frame
      */
-    public Rectangle calculateBodyTextFrame(List<ClassificationPage> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
+    private Rectangle calculateBodyTextFrame(List<ClassificationPage> pages,
+                                             FloatFrequencyCounter documentFontSizeCounter,
+                                             boolean landscape,
+                                             LayoutParsingType layoutParsingType) {
+
+        float approximateHeaderLineCount;
+        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
+            approximateHeaderLineCount = 3.3f;
+        } else {
+            approximateHeaderLineCount = 2.9f;
+        }
 
         BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();
 
@@ -86,8 +157,14 @@ public class BodyTextFrameService {
                         continue;
                     }
 
+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
+                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                        continue;
+                    }
+
                     float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (approxLineCount < APPROXIMATE_HEADER_LINE_COUNT) {
+                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10) || !layoutParsingType.equals(
+                            LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount) {
                         continue;
                     }
 
@@ -161,4 +238,4 @@ public class BodyTextFrameService {
 
     }
 
-}
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/GapsAcrossLinesService.java

@@ -6,6 +6,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
 import java.util.stream.Stream;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 
@@ -93,7 +94,7 @@ public class GapsAcrossLinesService {
         int lineCount = 1;
 
 
-        public GapAcrossLines(Rectangle2D rectangle2D) {
+        GapAcrossLines(Rectangle2D rectangle2D) {
 
             this.rectangle2D = correctRectangle(rectangle2D);
         }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PageContentExtractor.java

@@ -15,44 +15,44 @@ import org.springframework.core.io.ClassPathResource;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
-public class TextPositionSequenceSorter {
+public class PageContentExtractor {
 
-    public List<PageContents> getSortedTextPositionsWithPages(String filename) throws IOException {
+    public List<PageContents> getSortedPageContents(String filename) throws IOException {
 
         List<PageContents> textPositionSequencesPerPage = new LinkedList<>();
         try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
 
-            PDDocument pdDocument = Loader.loadPDF(inputStream);
+            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
 
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
 
-                PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-                PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-                stripper.setPageNumber(pageNumber);
-                stripper.setSortByPosition(true);
-                stripper.setStartPage(pageNumber);
-                stripper.setEndPage(pageNumber);
-                stripper.setPdpage(pdPage);
-                stripper.getText(pdDocument);
+                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+                    stripper.setPageNumber(pageNumber);
+                    stripper.setSortByPosition(true);
+                    stripper.setStartPage(pageNumber);
+                    stripper.setEndPage(pageNumber);
+                    stripper.setPdpage(pdPage);
+                    stripper.getText(pdDocument);
 
-                Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
-                        .stream()
-                        .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
+                    Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
+                            .stream()
+                            .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
 
-                var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());
+                    var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());
 
-                textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
-                        RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
-                        RectangleTransformations.toRectangle2D(pdPage.getMediaBox())));
+                    textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
+                            RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
+                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox()),
+                            stripper.getRulings()));
+                }
             }
-
-            pdDocument.close();
         }
 
         return textPositionSequencesPerPage;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PdfParsingService.java

@@ -1,151 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.parsing.PDFLinesTextStripper;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
-
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class PdfParsingService {
-
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final TaasBlockificationService taasBlockificationService;
-    private final DocuMineBlockificationService docuMineBlockificationService;
-    private final RedactManagerBlockificationService redactManagerBlockificationService;
-
-
-    public ClassificationDocument parseDocument(LayoutParsingType layoutParsingType,
-                                                PDDocument originDocument,
-                                                Map<Integer, List<TableCells>> pdfTableCells,
-                                                Map<Integer, List<ClassifiedImage>> pdfImages) {
-
-        ClassificationDocument document = new ClassificationDocument();
-        List<ClassificationPage> classificationPages = new ArrayList<>();
-
-        originDocument.setAllSecurityToBeRemoved(true);
-        long pageCount = originDocument.getNumberOfPages();
-
-        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-            parsePage(layoutParsingType, pdfImages, originDocument, pdfTableCells, document, classificationPages, pageNumber);
-        }
-
-        document.setPages(classificationPages);
-
-        return document;
-    }
-
-
-    @SneakyThrows
-    private void parsePage(LayoutParsingType layoutParsingType,
-                           Map<Integer, List<ClassifiedImage>> pdfImages,
-                           PDDocument pdDocument,
-                           Map<Integer, List<TableCells>> pdfTableCells,
-                           ClassificationDocument document,
-                           List<ClassificationPage> classificationPages,
-                           int pageNumber) {
-
-        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-        stripper.setPageNumber(pageNumber);
-        stripper.setStartPage(pageNumber);
-        stripper.setEndPage(pageNumber);
-        stripper.setPdpage(pdPage);
-        stripper.getText(pdDocument);
-
-        PDRectangle pdr = pdPage.getMediaBox();
-
-        int rotation = pdPage.getRotation();
-        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-        PDRectangle cropbox = pdPage.getCropBox();
-        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                stripper.getRulings(),
-                stripper.getMinCharWidth(),
-                stripper.getMaxCharHeight());
-
-        ClassificationPage classificationPage = switch (layoutParsingType) {
-            case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-        };
-
-        classificationPage.setRotation(rotation);
-        classificationPage.setLandscape(isLandscape);
-        classificationPage.setPageNumber(pageNumber);
-        classificationPage.setPageWidth(cropbox.getWidth());
-        classificationPage.setPageHeight(cropbox.getHeight());
-
-        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-            classificationPage.setImages(pdfImages.get(pageNumber));
-            imageServiceResponseAdapter.findOcr(classificationPage);
-        }
-
-        tableExtractionService.extractTables(cleanRulings, classificationPage);
-        buildPageStatistics(classificationPage);
-        increaseDocumentStatistics(classificationPage, document);
-
-        classificationPages.add(classificationPage);
-    }
-
-
-    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
-
-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-        }
-        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
-        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
-        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
-    }
-
-
-    private void buildPageStatistics(ClassificationPage classificationPage) {
-
-        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
-        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
-            if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getSequences() == null) {
-                    continue;
-                }
-                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
-                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
-                    classificationPage.getFontCounter().add(word.getFont());
-                    classificationPage.getFontSizeCounter().add(word.getFontSize());
-                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
-                }
-            }
-        }
-
-    }
-
-}
-
-

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/RulingCleaningService.java

@@ -12,7 +12,7 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;
@@ -53,7 +53,10 @@ public class RulingCleaningService {
         }
         List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);
 
-        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
+        return CleanRulings.builder()
+                .vertical(verticalRulingLines)
+                .horizontal(horizontalRulingLines)
+                .build();
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionGridCreatorService.java

@@ -20,11 +20,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 
 import lombok.RequiredArgsConstructor;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -0,0 +1,26 @@
+package com.knecon.fforesight.service.layoutparser.processor.services;
+
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+
+@Service
+public class SimplifiedSectionTextService {
+
+    public SimplifiedText toSimplifiedText(Document document) {
+
+        List<SimplifiedSectionText> simplifiedSectionTexts = document.getMainSections().stream().map(this::toSimplifiedSectionText).toList();
+        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build();
+    }
+
+
+    private SimplifiedSectionText toSimplifiedSectionText(Section section) {
+
+        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
+    }
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/TableExtractionService.java

@@ -12,6 +12,7 @@ import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -78,9 +79,9 @@ public class TableExtractionService {
      * @param cleanRulings The lines used to build the table.
      * @param page         Page object that contains textblocks and statistics.
      */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {
+    public void extractTables(CleanRulings cleanRulings, ClassificationPage page, LayoutParsingType layoutParsingType) {
 
-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
+        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical(), layoutParsingType);
 
         List<TextPageBlock> toBeRemoved = new ArrayList<>();
 
@@ -134,13 +135,16 @@ public class TableExtractionService {
     }
 
 
-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, LayoutParsingType layoutParsingType) {
 
-        for (Ruling r : horizontalRulingLines) {
-            if (r.getX2() < r.getX1()) {
-                double a = r.getX2();
-                r.x2 = (float) r.getX1();
-                r.x1 = (float) a;
+        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
+            // TODO: breaks some tables, for example "1 Abamectin Prr.pdf" try to fix this upstream in RulingCleaningService
+            for (Ruling r : horizontalRulingLines) {
+                if (r.getX2() < r.getX1()) {
+                    double a = r.getX2();
+                    r.x2 = (float) r.getX1();
+                    r.x1 = (float) a;
+                }
             }
         }