Native Build

Update dependency com.iqser.red.commons:storage-commons to v2.36.0 (main)

See merge request fforesight/layout-parser!50

.gitignore

@@ -18,6 +18,7 @@ target/
 .settings
 .springBeans
 .sts4-cache
+.gradle
 
 ### IntelliJ IDEA ###
 .idea
@@ -37,3 +38,7 @@ build/
 
 ### VS Code ###
 .vscode/
+gradlew.bat
+gradlew
+gradle.properties
+gradle/

.gitlab-ci.yml

@@ -1,4 +1,21 @@
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'
-    file: 'ci-templates/maven_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG

.mvn/wrapper/maven-wrapper.properties

@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.7/apache-maven-3.8.7-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar

buildSrc/build.gradle.kts

@@ -0,0 +1,7 @@
+plugins {
+    `kotlin-dsl`
+}
+
+repositories {
+    gradlePluginPortal()
+}

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -0,0 +1,75 @@
+plugins {
+    `java-library`
+    `maven-publish`
+    pmd
+    checkstyle
+    jacoco
+}
+
+group = "com.knecon.fforesight"
+
+java.sourceCompatibility = JavaVersion.VERSION_17
+java.targetCompatibility = JavaVersion.VERSION_17
+
+tasks.pmdMain {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
+}
+
+tasks.pmdTest {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/test_pmd.xml")
+}
+
+tasks.named<Test>("test") {
+    useJUnitPlatform()
+    reports {
+        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
+    }
+}
+
+tasks.test {
+    finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
+}
+
+tasks.jacocoTestReport {
+    dependsOn(tasks.test) // tests are required to run before generating the report
+    reports {
+        xml.required.set(true)
+        csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
+    }
+}
+
+allprojects {
+    publishing {
+        publications {
+            create<MavenPublication>(name) {
+                from(components["java"])
+            }
+        }
+        repositories {
+            maven {
+                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+                credentials {
+                    username = providers.gradleProperty("mavenUser").getOrNull();
+                    password = providers.gradleProperty("mavenPassword").getOrNull();
+                }
+            }
+        }
+    }
+}
+
+java {
+    withJavadocJar()
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven {
+        url = uri("https://nexus.knecon.com/repository/gindev/")
+        credentials {
+            username = providers.gradleProperty("mavenUser").getOrNull();
+            password = providers.gradleProperty("mavenPassword").getOrNull();
+        }
+    }
+}

config/checkstyle/checkstyle.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <property
+            name="severity"
+            value="error"/>
+    <module name="TreeWalker">
+        <module name="SuppressWarningsHolder"/>
+        <module name="MissingDeprecated"/>
+        <module name="MissingOverride"/>
+        <module name="AnnotationLocation"/>
+        <module name="JavadocStyle"/>
+        <module name="NonEmptyAtclauseDescription"/>
+        <module name="IllegalImport"/>
+        <module name="RedundantImport"/>
+        <module name="RedundantModifier"/>
+        <module name="EmptyBlock"/>
+        <module name="DefaultComesLast"/>
+        <module name="EmptyStatement"/>
+        <module name="EqualsHashCode"/>
+        <module name="ExplicitInitialization"/>
+        <module name="IllegalInstantiation"/>
+        <module name="ModifiedControlVariable"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="PackageDeclaration"/>
+        <module name="ParameterAssignment"/>
+        <module name="SimplifyBooleanExpression"/>
+        <module name="SimplifyBooleanReturn"/>
+        <module name="StringLiteralEquality"/>
+        <module name="OneStatementPerLine"/>
+        <module name="FinalClass"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="UpperEll"/>
+        <module name="OuterTypeFilename"/>
+    </module>
+    <module name="FileTabCharacter"/>
+    <module name="SuppressWarningsFilter"/>
+</module>

config/pmd/pmd.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon ruleset checks the code for bad stuff
+    </description>
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+    </rule>
+
+</ruleset>

config/pmd/test_pmd.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon test ruleset checks the code for bad stuff
+    </description>
+
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="AvoidFieldNameMatchingTypeName"/>
+        <exclude name="TestClassWithoutTestCases"/>
+    </rule>
+
+</ruleset>

gradle.properties.kts

@@ -0,0 +1 @@
+version = 0.1-SNAPSHOT

layoutparser-service-image/pom.xml

@@ -1,100 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>platform-docker-dependency</artifactId>
-        <version>0.1.0</version>
-        <relativePath/>
-    </parent>
-
-    <groupId>com.knecon.fforesight</groupId>
-    <artifactId>layoutparser-service-image</artifactId>
-    <version>1.0.0</version>
-    <packaging>pom</packaging>
-
-    <properties>
-        <service.server>layoutparser-service-server</service.server>
-        <platform.jar>${service.server}.jar</platform.jar>
-        <docker.skip.push>false</docker.skip.push>
-        <docker.image.prefix>ff</docker.image.prefix>
-        <docker.image.name>${docker.image.prefix}/${service.server}</docker.image.name>
-    </properties>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-dependency-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-resources-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>io.fabric8</groupId>
-                <artifactId>docker-maven-plugin</artifactId>
-            </plugin>
-        </plugins>
-
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-dependency-plugin</artifactId>
-                    <executions>
-                        <execution>
-                            <id>download-platform-jar</id>
-                            <phase>prepare-package</phase>
-                            <goals>
-                                <goal>copy</goal>
-                            </goals>
-                            <configuration>
-                                <artifactItems>
-                                    <dependency>
-                                        <groupId>${project.groupId}</groupId>
-                                        <artifactId>${service.server}</artifactId>
-                                        <version>${project.version}</version>
-                                        <type>jar</type>
-                                        <overWrite>true</overWrite>
-                                        <destFileName>${platform.jar}</destFileName>
-                                    </dependency>
-                                </artifactItems>
-                                <outputDirectory>${docker.build.directory}</outputDirectory>
-                            </configuration>
-                        </execution>
-                    </executions>
-                </plugin>
-                <plugin>
-                    <groupId>io.fabric8</groupId>
-                    <artifactId>docker-maven-plugin</artifactId>
-                    <configuration>
-                        <images>
-                            <image>
-                                <name>${docker.image.name}</name>
-                                <build>
-                                    <dockerFileDir>${docker.build.directory}</dockerFileDir>
-                                    <args>
-                                        <PLATFORM_JAR>${platform.jar}</PLATFORM_JAR>
-                                    </args>
-                                    <tags>
-                                        <tag>${docker.image.version}</tag>
-                                        <tag>latest</tag>
-                                    </tags>
-                                </build>
-                            </image>
-                        </images>
-                    </configuration>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
-</project>

layoutparser-service-image/src/main/docker/Dockerfile

@@ -1,9 +0,0 @@
-FROM red/base-image:2.0.2
-
-ARG PLATFORM_JAR
-
-ENV PLATFORM_JAR ${PLATFORM_JAR}
-
-ENV USES_ELASTICSEARCH false
-
-COPY ["${PLATFORM_JAR}", "/"]

layoutparser-service/layoutparser-service-internal-api/build.gradle.kts

@@ -0,0 +1,6 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.2.2"
+}
+
+description = "layoutparser-service-internal-api"

layoutparser-service/layoutparser-service-internal-api/pom.xml

@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>1.0.0</version>
-    </parent>
-
-    <artifactId>layoutparser-service-internal-api</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>${guava.version}</version>
-        </dependency>
-    </dependencies>
-
-</project>

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentData.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -12,10 +12,9 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DocumentData {
 
-    PageData[] pages;
-    AtomicTextBlockData[] atomicTextBlocks;
-    AtomicPositionBlockData[] atomicPositionBlocks;
-    DocumentTreeData documentTreeData;
-
+    DocumentPage[] documentPages;
+    DocumentTextData[] documentTextData;
+    DocumentPositionData[] documentPositions;
+    DocumentStructure documentStructure;
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPage.java

@@ -1,16 +1,18 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class PageData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentPage {
 
     int number;
     int height;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentPositionData.java

@@ -1,16 +1,18 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class AtomicPositionBlockData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentPositionData {
 
     Long id;
     int[] stringIdxToPositionIdx;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentStructure.java

@@ -1,5 +1,7 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
+import java.awt.geom.Rectangle2D;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -8,20 +10,53 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
-@AllArgsConstructor
 @NoArgsConstructor
+@AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentTreeData {
+public class DocumentStructure {
 
     EntryData root;
 
 
+    public static class TableProperties {
+
+        public static final String NUMBER_OF_ROWS = "numberOfRows";
+        public static final String NUMBER_OF_COLS = "numberOfCols";
+
+    }
+
+    public static class ImageProperties {
+
+        public static final String TRANSPARENT = "transparent";
+        public static final String IMAGE_TYPE = "imageType";
+        public static final String POSITION = "position";
+        public static final String ID = "id";
+
+    }
+
+    public static class TableCellProperties {
+
+        public static final String B_BOX = "bBox";
+        public static final String ROW = "row";
+        public static final String COL = "col";
+        public static final String HEADER = "header";
+
+    }
+
+    public static final String RECTANGLE_DELIMITER = ";";
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+
     public EntryData get(List<Integer> tocId) {
 
         if (tocId.isEmpty()) {
@@ -37,7 +72,7 @@ public class DocumentTreeData {
 
     public Stream<EntryData> streamAllEntries() {
 
-        return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentTreeData::flatten);
+        return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentStructure::flatten);
     }
 
 
@@ -49,14 +84,15 @@ public class DocumentTreeData {
 
     private static Stream<EntryData> flatten(EntryData entry) {
 
-        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTreeData::flatten));
+        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentStructure::flatten));
     }
 
 
+    @Data
     @Builder
-    @Getter
+    @NoArgsConstructor
     @AllArgsConstructor
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    @FieldDefaults(level = AccessLevel.PRIVATE)
     public static class EntryData {
 
         NodeType type;
@@ -66,6 +102,7 @@ public class DocumentTreeData {
         Map<String, String> properties;
         List<EntryData> children;
 
+
         @Override
         public String toString() {
 

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/DocumentTextData.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 
 
@@ -6,13 +6,15 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class AtomicTextBlockData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentTextData {
 
     Long id;
     Long page;
@@ -22,6 +24,4 @@ public class AtomicTextBlockData {
     int end;
     int[] lineBreaks;
 
-
-
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/NodeType.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import java.util.Locale;
 

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedSectionText.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.text;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.text;
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 import java.util.ArrayList;
 import java.util.List;

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/ParagraphData.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class ParagraphData {
+
+    private String text;
+    List<Range> boldTextBoundaries;
+    List<Range> italicTextBoundaries;
+    List<Integer> linebreaks;
+    private String classification;
+
+    private String orientation;
+    private int textDirection;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/Range.java

@@ -0,0 +1,5 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+public record Range(int start, int end) {
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/ResearchDocumentData.java

@@ -0,0 +1,16 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+@Builder
+@Data
+@AllArgsConstructor
+public class ResearchDocumentData {
+
+    String originalFile;
+    List<StructureObject> structureObjects;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/RowData.java

@@ -0,0 +1,15 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class RowData {
+
+    boolean header;
+    List<ParagraphData> cellText;
+    float[] bBox;
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/StructureObject.java

@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+@AllArgsConstructor
+public class StructureObject {
+
+    Integer structureObjectNumber;
+    int page;
+    int stringOffset;
+    float[] boundingBox;
+    ParagraphData paragraph;
+    TableData table;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/taas/TableData.java

@@ -0,0 +1,16 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.taas;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class TableData {
+
+    List<RowData> rowData;
+    Integer numberOfCols;
+    Integer numberOfRows;
+
+}

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "LAYOUTPARSING_REQUEST_QUEUE";
-    public static final String LAYOUT_PARSING_DLQ = "LAYOUTPARSING_DLQ";
-    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "LAYOUTPARSING_FINISHED_EVENT_QUEUE";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
+    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_dead_letter_queue";
+    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -4,17 +4,22 @@ import java.util.Map;
 import java.util.Optional;
 
 import lombok.Builder;
-
+import lombok.NonNull;
 
 @Builder
 public record LayoutParsingRequest(
+        @NonNull LayoutParsingType layoutParsingType,
         Map<String, String> identifier,
-        String originFileStorageId,
+        @NonNull String originFileStorageId,
         Optional<String> tablesFileStorageId,
         Optional<String> imagesFileStorageId,
-        String structureFileStorageId,
-        String textBlockFileStorageId,
-        String positionBlockFileStorageId,
-        String pageFileStorageId) {
+        @NonNull String structureFileStorageId,
+        String researchDocumentStorageId,
+        @NonNull String textBlockFileStorageId,
+        @NonNull String positionBlockFileStorageId,
+        @NonNull String pageFileStorageId,
+        @NonNull String simplifiedTextStorageId,
+        @NonNull String viewerDocumentStorageId,
+        @NonNull String sectionGridStorageId) {
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java

@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.queue;
+
+public enum LayoutParsingType {
+    REDACT_MANAGER,
+    TAAS,
+    DOCUMINE
+}

layoutparser-service/layoutparser-service-internal-api/src/test/resources/log4j2-test.xml

@@ -0,0 +1,16 @@
+<Configuration>
+
+    <Appenders>
+        <Console name="CONSOLE" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="warn">
+            <AppenderRef ref="CONSOLE"/>
+        </Root>
+        <Logger name="com.iqser" level="info"/>
+    </Loggers>
+
+</Configuration>

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -0,0 +1,26 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.2.2"
+}
+
+description = "layoutparser-service-processor"
+
+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0-RC1"
+
+dependencies {
+    implementation(project(":layoutparser-service-internal-api"))
+
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.138.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
+    }
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("com.iqser.red.commons:storage-commons:2.36.0")
+
+    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
+    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.1.2")
+}

layoutparser-service/layoutparser-service-processor/pom.xml

@@ -1,65 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>1.0.0</version>
-    </parent>
-
-    <artifactId>layoutparser-service-processor</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-shared-api-v1</artifactId>
-            <version>2.36.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>tenant-commons</artifactId>
-            <version>${tennat-commons.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>layoutparser-service-internal-api</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>storage-commons</artifactId>
-            <version>${storage-commons.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox-tools</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-afterburner</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-jsr310</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-web</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-amqp</artifactId>
-        </dependency>
-    </dependencies>
-
-</project>

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -0,0 +1,251 @@
+package com.knecon.fforesight.service.layoutparser.processor;
+
+import static java.lang.String.format;
+
+import java.awt.geom.Rectangle2D;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
+import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class LayoutParsingPipeline {
+
+    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
+    private final CvTableParsingAdapter cvTableParsingAdapter;
+    private final LayoutParsingStorageService layoutParsingStorageService;
+    private final SectionsBuilderService sectionsBuilderService;
+    private final SectionGridCreatorService sectionGridCreatorService;
+    private final TaasClassificationService taasClassificationService;
+    private final RedactManagerClassificationService redactManagerClassificationService;
+    private final DocuMineClassificationService docuMineClassificationService;
+    private final SimplifiedSectionTextService simplifiedSectionTextService;
+    private final BodyTextFrameService bodyTextFrameService;
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;
+    private final ViewerDocumentService viewerDocumentService;
+
+
+    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
+
+        long start = System.currentTimeMillis();
+
+        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
+            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            }
+
+            TableServiceResponse tableServiceResponse = new TableServiceResponse();
+            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+            }
+
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+
+            int numberOfPages = originDocument.getNumberOfPages();
+
+            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
+            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+            try (var out = new ByteArrayOutputStream()) {
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out);
+                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
+            }
+
+            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+            }
+
+            return LayoutParsingFinishedEvent.builder()
+                    .identifier(layoutParsingRequest.identifier())
+                    .numberOfPages(numberOfPages)
+                    .duration(System.currentTimeMillis() - start)
+                    .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
+                            layoutParsingRequest.structureFileStorageId(),
+                            layoutParsingRequest.textBlockFileStorageId(),
+                            layoutParsingRequest.positionBlockFileStorageId(),
+                            layoutParsingRequest.pageFileStorageId()))
+                    .build();
+        }
+    }
+
+
+    @SneakyThrows
+    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
+                                              PDDocument originDocument,
+                                              ImageServiceResponse imageServiceResponse,
+                                              TableServiceResponse tableServiceResponse) {
+
+        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
+        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
+
+        ClassificationDocument classificationDocument = new ClassificationDocument();
+        List<ClassificationPage> classificationPages = new ArrayList<>();
+
+        originDocument.setAllSecurityToBeRemoved(true);
+        long pageCount = originDocument.getNumberOfPages();
+
+        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+            classificationDocument.setPages(classificationPages);
+            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+            PDPage pdPage = originDocument.getPage(pageNumber - 1);
+            stripper.setPageNumber(pageNumber);
+            stripper.setStartPage(pageNumber);
+            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
+            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE)) {
+                stripper.setSortByPosition(true);
+            }
+            stripper.getText(originDocument);
+
+            PDRectangle pdr = pdPage.getMediaBox();
+
+            int rotation = pdPage.getRotation();
+            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+            PDRectangle cropbox = pdPage.getCropBox();
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                    stripper.getRulings(),
+                    stripper.getMinCharWidth(),
+                    stripper.getMaxCharHeight());
+
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER ->
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS ->
+                        taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE ->
+                        docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            };
+            classificationPage.setCleanRulings(cleanRulings);
+            classificationPage.setRotation(rotation);
+            classificationPage.setLandscape(isLandscape);
+            classificationPage.setPageNumber(pageNumber);
+            classificationPage.setPageWidth(cropbox.getWidth());
+            classificationPage.setPageHeight(cropbox.getHeight());
+
+            // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
+            classificationPage.setMarkedContentBboxPerType(convertMarkedContents(stripper.getMarkedContents()));
+
+            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+                classificationPage.setImages(pdfImages.get(pageNumber));
+                imageServiceResponseAdapter.findOcr(classificationPage);
+            }
+
+            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+            buildPageStatistics(classificationPage);
+            increaseDocumentStatistics(classificationPage, classificationDocument);
+
+            classificationPages.add(classificationPage);
+        }
+
+        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+
+        switch (layoutParsingType) {
+            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
+            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
+        }
+
+        sectionsBuilderService.buildSections(classificationDocument);
+        sectionsBuilderService.addImagesToSections(classificationDocument);
+
+        return classificationDocument;
+    }
+
+
+    private Map<String, Rectangle2D> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
+        Map<String, Rectangle2D> markedContentBboxes = new HashMap<>();
+        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
+        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
+        return markedContentBboxes;
+    }
+
+    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
+
+        if (!classificationPage.isLandscape()) {
+            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        }
+        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
+        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
+        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
+    }
+
+
+    private void buildPageStatistics(ClassificationPage classificationPage) {
+
+        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
+        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
+                    continue;
+                }
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
+                    classificationPage.getFontCounter().add(word.getFont());
+                    classificationPage.getFontSizeCounter().add(word.getFontSize());
+                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
+                }
+            }
+        }
+
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingService.java

@@ -1,87 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor;
-
-import static java.lang.String.format;
-
-import java.io.IOException;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.ClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.PdfParsingService;
-import com.knecon.fforesight.service.layoutparser.processor.classification.service.SectionsBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.mapper.DocumentDataMapper;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class LayoutParsingService {
-
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-    private final CvTableParsingAdapter cvTableParsingAdapter;
-    private final LayoutParsingStorageService layoutParsingStorageService;
-    private final PdfParsingService pdfParsingService;
-    private final ClassificationService classificationService;
-    private final SectionsBuilderService sectionsBuilderService;
-
-
-    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
-
-        long start = System.currentTimeMillis();
-        PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-
-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
-        }
-
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
-        }
-
-        Document documentGraph = parseLayout(originDocument, imageServiceResponse, tableServiceResponse);
-        int numberOfPages = originDocument.getNumberOfPages();
-        originDocument.close();
-
-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-
-        return LayoutParsingFinishedEvent.builder()
-                .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(numberOfPages)
-                .duration(System.currentTimeMillis() - start)
-                .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId()))
-                .build();
-    }
-
-
-    public Document parseLayout(PDDocument originDocument, ImageServiceResponse imageServiceResponse, TableServiceResponse tableServiceResponse) {
-
-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(originDocument,
-                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
-                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
-
-        classificationService.classifyDocument(classificationDocument);
-
-        sectionsBuilderService.buildSections(classificationDocument);
-
-        return DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -1,10 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
@@ -13,18 +18,18 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicTextBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.DocumentTreeData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.PageData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -38,10 +43,11 @@ public class LayoutParsingStorageService {
 
     public PDDocument getOriginFile(String storageId) throws IOException {
 
-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var originDocumentInputStream = getObject(storageId)) {
             File tempFile = createTempFile("document", ".pdf");
             try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
                 IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
             }
             return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
         }
@@ -50,52 +56,44 @@ public class LayoutParsingStorageService {
 
     public ImageServiceResponse getImagesFile(String storageId) throws IOException {
 
-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (InputStream inputStream = getObject(storageId)) {
 
-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
         }
     }
 
 
     public TableServiceResponse getTablesFile(String storageId) throws IOException {
 
-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+        try (var tableClassificationStream = getObject(storageId)) {
 
+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
         }
     }
 
 
-    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) throws IOException {
-
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentTreeData());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getAtomicTextBlocks());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getAtomicPositionBlocks());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getPages());
+    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {
 
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getDocumentTextData());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getDocumentPositions());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getDocumentPages());
     }
 
 
-    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
+    public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) {
 
-        PageData[] pageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), PageData[].class);
-        AtomicTextBlockData[] atomicTextBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.textBlockFileStorageId(),
-                AtomicTextBlockData[].class);
-        AtomicPositionBlockData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.positionBlockFileStorageId(),
-                AtomicPositionBlockData[].class);
-        DocumentTreeData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.structureFileStorageId(),
-                DocumentTreeData.class);
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid);
+    }
 
-        return DocumentData.builder()
-                .documentTreeData(tableOfContentsData)
-                .atomicPositionBlocks(atomicPositionBlockData)
-                .atomicTextBlocks(atomicTextBlockData)
-                .pages(pageData)
-                .build();
+
+    public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.researchDocumentStorageId(), researchDocumentData);
     }
 
 
@@ -123,4 +121,30 @@ public class LayoutParsingStorageService {
         }
     }
 
+
+    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
+    }
+
+
+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
+
+    @SneakyThrows
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {
+
+        try (var in = new ByteArrayInputStream(out.toByteArray())) {
+
+            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
+        }
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Classification.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import lombok.Data;
-
-@Data
-public class Classification {
-
-    private Map<String, Float> probabilities = new HashMap<>();
-    private String label;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/FilterGeometry.java

@@ -1,11 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class FilterGeometry {
-
-    private ImageSize imageSize;
-    private ImageFormat imageFormat;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Filters.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Filters {
-
-    private FilterGeometry geometry;
-    private Probability probability;
-    private boolean allPassed;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Geometry.java

@@ -1,11 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Geometry {
-
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageFormat.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageFormat {
-
-    private float quotient;
-    private boolean tooTall;
-    private boolean tooWide;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageMetadata.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageMetadata {
-
-    private Classification classification;
-    private Position position;
-    private Geometry geometry;
-    private Filters filters;
-    private boolean alpha;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/ImageSize.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class ImageSize {
-
-    private float quotient;
-    private boolean tooLarge;
-    private boolean tooSmall;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Position.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Position {
-
-    private float x1;
-    private float x2;
-    private float y1;
-    private float y2;
-    private int pageNumber;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/image/Probability.java

@@ -1,10 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
-
-import lombok.Data;
-
-@Data
-public class Probability {
-
-    private boolean unconfident;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/PageInfo.java

@@ -1,13 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import lombok.Data;
-
-@Data
-public class PageInfo {
-
-    private int number;
-    private int rotation;
-    private float width;
-    private float height;
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/adapter/model/table/TableData.java

@@ -1,14 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import lombok.Data;
-
-@Data
-public class TableData {
-
-    private PageInfo pageInfo;
-    private List<TableCells> tableCells = new ArrayList<>();
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/model/Orientation.java

@@ -1,8 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
-
-public enum Orientation {
-
-    NONE,
-    LEFT,
-    RIGHT
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/service/PdfParsingService.java

@@ -1,136 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.service;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.classification.parsing.PDFLinesTextStripper;
-
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class PdfParsingService {
-
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final BlockificationService blockificationService;
-    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
-
-
-    public ClassificationDocument parseDocument(PDDocument originDocument, Map<Integer, List<TableCells>> pdfTableCells, Map<Integer, List<ClassifiedImage>> pdfImages) {
-
-        ClassificationDocument document = new ClassificationDocument();
-        List<ClassificationPage> classificationPages = new ArrayList<>();
-
-        originDocument.setAllSecurityToBeRemoved(true);
-        long pageCount = originDocument.getNumberOfPages();
-
-        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-            parsePage(pdfImages, originDocument, pdfTableCells, document, classificationPages, pageNumber);
-        }
-
-        document.setPages(classificationPages);
-
-        return document;
-    }
-
-
-    @SneakyThrows
-    private void parsePage(Map<Integer, List<ClassifiedImage>> pdfImages,
-                           PDDocument pdDocument,
-                           Map<Integer, List<TableCells>> pdfTableCells,
-                           ClassificationDocument document,
-                           List<ClassificationPage> classificationPages,
-                           int pageNumber) {
-
-        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-        stripper.setPageNumber(pageNumber);
-        stripper.setStartPage(pageNumber);
-        stripper.setEndPage(pageNumber);
-        stripper.setPdpage(pdPage);
-        stripper.getText(pdDocument);
-
-        PDRectangle pdr = pdPage.getMediaBox();
-
-        int rotation = pdPage.getRotation();
-        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-        PDRectangle cropbox = pdPage.getCropBox();
-        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                stripper.getRulings(),
-                stripper.getMinCharWidth(),
-                stripper.getMaxCharHeight());
-        ClassificationPage classificationPage = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-        classificationPage.setRotation(rotation);
-        classificationPage.setLandscape(isLandscape);
-        classificationPage.setPageNumber(pageNumber);
-        classificationPage.setPageWidth(cropbox.getWidth());
-        classificationPage.setPageHeight(cropbox.getHeight());
-
-        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-            classificationPage.setImages(pdfImages.get(pageNumber));
-            imageServiceResponseAdapter.findOcr(classificationPage);
-        }
-
-        tableExtractionService.extractTables(cleanRulings, classificationPage);
-        buildPageStatistics(classificationPage);
-        increaseDocumentStatistics(classificationPage, document);
-
-        classificationPages.add(classificationPage);
-    }
-
-
-    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
-
-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-        }
-        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
-        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
-        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
-    }
-
-
-    private void buildPageStatistics(ClassificationPage classificationPage) {
-
-        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
-        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
-            if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getSequences() == null) {
-                    continue;
-                }
-                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
-                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
-                    classificationPage.getFontCounter().add(word.getFont());
-                    classificationPage.getFontSizeCounter().add(word.getFontSize());
-                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
-                }
-            }
-        }
-
-    }
-
-}
-
-

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/classification/utils/FileUtils.java

@@ -1,56 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.utils;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-
-import lombok.experimental.UtilityClass;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@UtilityClass
-public class FileUtils {
-
-    public File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {
-
-        File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();
-        setRWPermissionsOnlyForOwner(tempFile);
-
-        return tempFile;
-    }
-
-
-    /**
-     * Deletes a file; logs a message with the reason if the deletion fails.
-     * This method is null-safe.
-     *
-     * @param file The file to delete.  Can be null.
-     */
-    public void deleteFile(File file) {
-
-        if (file != null) {
-            try {
-                Files.deleteIfExists(file.toPath());
-            } catch (IOException ex) {
-                log.warn("Could not delete file!", ex);
-            }
-        }
-    }
-
-
-    // We don't need to check the results of the permission setters below,
-    // since we're manipulating a file we created ourselves.
-    @SuppressWarnings({"ResultOfMethodCallIgnored", "squid:S899"})
-    private void setRWPermissionsOnlyForOwner(File tempFile) {
-
-        try {
-            tempFile.setReadable(true, true);
-            tempFile.setWritable(true, true);
-            tempFile.setExecutable(false);
-        } catch (SecurityException ex) {
-            // This should never happen since we're creating a temp file ourselves.
-            log.warn("Caught an exception during temp file creation.  This should not happend.  Check the code.", ex);
-        }
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/graph/nodes/GenericSemanticNode.java

@@ -1,5 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
-
-public interface GenericSemanticNode extends SemanticNode {
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/mapper/PropertiesMapper.java

@@ -1,112 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.mapper;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.services.RectangleTransformations;
-
-public class PropertiesMapper {
-
-    public static final String TRANSPARENT = "transparent";
-    public static final String IMAGE_TYPE = "imageType";
-    public static final String POSITION = "position";
-    public static final String ROW = "row";
-    public static final String COL = "col";
-    public static final String HEADER = "header";
-    public static final String B_BOX = "bBox";
-    public static final String NUMBER_OF_ROWS = "numberOfRows";
-    public static final String NUMBER_OF_COLS = "numberOfCols";
-
-
-    public static Map<String, String> buildImageProperties(Image image) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put(IMAGE_TYPE, image.getImageType().toString());
-        properties.put(TRANSPARENT, String.valueOf(image.isTransparent()));
-        properties.put(POSITION, RectangleTransformations.toString(image.getPosition()));
-        return properties;
-    }
-
-
-    public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put(ROW, String.valueOf(tableCell.getRow()));
-        properties.put(COL, String.valueOf(tableCell.getCol()));
-        properties.put(HEADER, String.valueOf(tableCell.isHeader()));
-
-        if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
-            throw new IllegalArgumentException("TableCell can only occur on a single page!");
-        }
-        String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
-        properties.put(B_BOX, bBoxString);
-
-        return properties;
-    }
-
-
-    public static Map<String, String> buildTableProperties(Table table) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put(NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
-        properties.put(NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
-        return properties;
-    }
-
-
-    public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
-
-        builder.imageType(parseImageType(properties.get(IMAGE_TYPE)));
-        builder.transparent(Boolean.parseBoolean(properties.get(TRANSPARENT)));
-        builder.position(parseRectangle2D(properties.get(POSITION)));
-    }
-
-
-    public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
-
-        builder.row(Integer.parseInt(properties.get(ROW)));
-        builder.col(Integer.parseInt(properties.get(COL)));
-        builder.header(Boolean.parseBoolean(properties.get(HEADER)));
-        builder.bBox(parseRectangle2D(properties.get(B_BOX)));
-    }
-
-
-    public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
-
-        builder.numberOfRows(Integer.parseInt(properties.get(NUMBER_OF_ROWS)));
-        builder.numberOfCols(Integer.parseInt(properties.get(NUMBER_OF_COLS)));
-    }
-
-
-    private static ImageType parseImageType(String imageType) {
-
-        return switch (imageType) {
-            case "LOGO" -> ImageType.LOGO;
-            case "FORMULA" -> ImageType.FORMULA;
-            case "SIGNATURE" -> ImageType.SIGNATURE;
-            case "OCR" -> ImageType.OCR;
-            default -> ImageType.OTHER;
-        };
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return String.format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/AbstractPageBlock.java

@@ -1,8 +1,8 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationDocument.java

@@ -1,11 +1,11 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.ArrayList;
 import java.util.List;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.StringFrequencyCounter;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.UnclassifiedText;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText;
 
 import lombok.Data;
 import lombok.NoArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationFooter.java

@@ -1,8 +1,8 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationHeader.java

@@ -1,8 +1,8 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java

@@ -1,15 +1,20 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.StringFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 
 @Data
 @RequiredArgsConstructor
@@ -35,4 +40,8 @@ public class ClassificationPage {
     private float pageWidth;
     private float pageHeight;
 
+    CleanRulings cleanRulings;
+
+    private Map<String, Rectangle2D> markedContentBboxPerType = new HashMap<>();
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationSection.java

@@ -1,10 +1,10 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.ArrayList;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 
 import lombok.Data;
 import lombok.NoArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/FloatFrequencyCounter.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 import java.util.ArrayList;
 import java.util.Collections;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/GapInformation.java

@@ -0,0 +1,27 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class GapInformation {
+
+    List<List<Rectangle2D>> xGaps;
+    List<List<Rectangle2D>> yGaps;
+
+
+    public GapInformation() {
+
+        xGaps = new LinkedList<>();
+        yGaps = new LinkedList<>();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/LineInformation.java

@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class LineInformation {
+
+    List<Rectangle2D> lineBBox;
+    List<List<TextPositionSequence>> sequencesByLines;
+    List<List<Rectangle2D>> bBoxWithGapsByLines;
+    List<List<List<TextPositionSequence>>> sequencesWithGapsByLines;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/Orientation.java

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+public enum Orientation {
+
+    NONE,
+    LEFT,
+    RIGHT
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageBlockType.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model;
+package com.knecon.fforesight.service.layoutparser.processor.model;
 
 public enum PageBlockType {
     H1,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageContents.java

@@ -0,0 +1,22 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+
+@Getter
+@Builder
+@AllArgsConstructor
+public class PageContents {
+
+    List<TextPositionSequence> sortedTextPositionSequences;
+    Rectangle2D cropBox;
+    Rectangle2D mediaBox;
+    List<Ruling> rulings;
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageInformation.java

@@ -0,0 +1,17 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.awt.geom.Rectangle2D;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@Getter
+@AllArgsConstructor
+public class PageInformation {
+
+    PageContents pageContents;
+    LineInformation lineInformation;
+    Rectangle2D mainBodyTextFrame;
+    GapInformation gapInformation;
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/SectionIdentifier.java

@@ -0,0 +1,123 @@
+package com.knecon.fforesight.service.layoutparser.processor.model;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class SectionIdentifier {
+
+    static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+
+    private enum Format {
+        EMPTY,
+        NUMERICAL,
+        DOCUMENT
+    }
+
+    Format format;
+    String identifierString;
+    List<Integer> identifiers;
+    boolean asChild;
+
+
+    public static SectionIdentifier fromSearchText(String headline) {
+
+        if (headline == null || headline.isEmpty() || headline.isBlank()) {
+            return SectionIdentifier.empty();
+        }
+
+        Matcher numericalIdentifierMatcher = numericalIdentifierPattern.matcher(headline);
+        if (numericalIdentifierMatcher.find()) {
+            return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher);
+        }
+        // more formats here
+        return SectionIdentifier.empty();
+    }
+
+
+    public static SectionIdentifier asChildOf(SectionIdentifier sectionIdentifier) {
+
+        return new SectionIdentifier(sectionIdentifier.format, sectionIdentifier.toString(), sectionIdentifier.identifiers, true);
+    }
+
+
+    public static SectionIdentifier document() {
+
+        return new SectionIdentifier(Format.DOCUMENT, "document", Collections.emptyList(), false);
+    }
+
+
+    public static SectionIdentifier empty() {
+
+        return new SectionIdentifier(Format.EMPTY, "empty", Collections.emptyList(), false);
+    }
+
+
+    private static SectionIdentifier buildNumericalSectionIdentifier(String headline, Matcher numericalIdentifierMatcher) {
+
+        String identifierString = headline.substring(numericalIdentifierMatcher.start(), numericalIdentifierMatcher.end());
+        List<Integer> identifiers = new LinkedList<>();
+        for (int i = 1; i <= 4; i++) {
+            String numericalIdentifier = numericalIdentifierMatcher.group(i);
+            if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) {
+                break;
+            }
+            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
+        }
+        return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false);
+    }
+
+
+    /**
+     * Determines if the current section is the parent of the given section.
+     *
+     * @param sectionIdentifier The section identifier to compare against.
+     * @return true if the current section is the parent of the given section, false otherwise.
+     */
+    public boolean isParentOf(SectionIdentifier sectionIdentifier) {
+
+        if (this.format.equals(Format.EMPTY)) {
+            return false;
+        }
+        if (this.format.equals(Format.DOCUMENT)) {
+            return true;
+        }
+        if (!this.format.equals(sectionIdentifier.format)) {
+            return false;
+        }
+        if (this.identifiers.size() >= sectionIdentifier.identifiers.size() && !(this.identifiers.size() == sectionIdentifier.identifiers.size() && sectionIdentifier.asChild)) {
+            return false;
+        }
+        for (int i = 0; i < this.identifiers.size(); i++) {
+            if (!this.identifiers.get(i).equals(sectionIdentifier.identifiers.get(i))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    public boolean isChildOf(SectionIdentifier sectionIdentifier) {
+
+        if (this.format.equals(Format.DOCUMENT) || this.format.equals(Format.EMPTY)) {
+            return false;
+        }
+        return sectionIdentifier.isParentOf(this);
+    }
+
+
+    @Override
+    public String toString() {
+
+        return identifierString;
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/Boundary.java

@@ -1,12 +1,11 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph;
 
 import static java.lang.String.format;
 
 import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import java.util.stream.IntStream;
 
 import lombok.EqualsAndHashCode;
 import lombok.Setter;
@@ -109,6 +108,10 @@ public class Boundary implements Comparable<Boundary> {
         return splitBoundaries;
     }
 
+    public IntStream intStream() {
+
+        return IntStream.range(start, end);
+    }
 
     public static Boundary merge(Collection<Boundary> boundaries) {
 
@@ -138,26 +141,4 @@ public class Boundary implements Comparable<Boundary> {
         return 0;
     }
 
-
-    /**
-     * shrinks the boundary, such that textBlock.subSequence(boundary) returns a string without whitespaces.
-     *
-     * @param textBlock TextBlock to check whitespaces against
-     * @return boundary
-     */
-    public Boundary trim(TextBlock textBlock) {
-
-        int trimmedStart = this.start;
-        while (Character.isWhitespace(textBlock.charAt(trimmedStart))) {
-            trimmedStart++;
-        }
-
-        int trimmedEnd = this.end;
-        while (Character.isWhitespace(textBlock.charAt(trimmedEnd - 1))) {
-            trimmedEnd--;
-        }
-
-        return new Boundary(trimmedStart, Math.max(trimmedEnd, trimmedStart));
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/DocumentTree.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph;
 
 import static java.lang.String.format;
 
@@ -7,14 +7,14 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/entity/EntityType.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
 
 public enum EntityType {
     ENTITY,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/entity/RedactionEntity.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
@@ -11,9 +11,9 @@ import java.util.Map;
 import java.util.Set;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 
 import lombok.AccessLevel;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/entity/RedactionPosition.java

@@ -1,9 +1,9 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
 
 import java.awt.geom.Rectangle2D;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -11,11 +11,11 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.amazonaws.services.kms.model.NotFoundException;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Footer.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/GenericSemanticNode.java

@@ -0,0 +1,5 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+
+public interface GenericSemanticNode extends SemanticNode {
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Header.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Headline.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Image.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -8,11 +8,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/ImageType.java

@@ -1,4 +1,6 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+
+import java.util.Locale;
 
 public enum ImageType {
     LOGO,
@@ -10,7 +12,7 @@ public enum ImageType {
 
     public static ImageType fromString(String imageType) {
 
-        return switch (imageType.toLowerCase()) {
+        return switch (imageType.toLowerCase(Locale.ROOT)) {
             case "logo" -> ImageType.LOGO;
             case "formula" -> ImageType.FORMULA;
             case "signature" -> ImageType.SIGNATURE;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Page.java

@@ -1,14 +1,14 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Paragraph.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Section.java

@@ -1,14 +1,14 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/SemanticNode.java

@@ -1,23 +1,25 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.EntityType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.RectangleTransformations;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.EntityType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 public interface SemanticNode {
 
@@ -59,6 +61,12 @@ public interface SemanticNode {
     }
 
 
+    default Page getFirstPage() {
+
+        return getTextBlock().getPages().stream().min(Comparator.comparingInt(Page::getNumber)).orElseThrow(() -> new IllegalStateException("SemanticNode has no Page!"));
+    }
+
+
     /**
      * Each AtomicTextBlock is assigned a page, so to get the pages for this boundary, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock.
      *
@@ -278,7 +286,7 @@ public interface SemanticNode {
      */
     default boolean containsStringIgnoreCase(String string) {
 
-        return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
+        return getTextBlock().getSearchText().toLowerCase(Locale.ROOT).contains(string.toLowerCase(Locale.ROOT));
     }
 
 
@@ -306,7 +314,6 @@ public interface SemanticNode {
     }
 
 
-
     /**
      * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the RedactionEntity intersects or even contains the RedactionEntity.
      * It sets the fields accordingly and recursively calls this function on all its children.

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java

@@ -1,19 +1,20 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import static java.lang.String.format;
 
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -67,7 +68,7 @@ public class Table implements SemanticNode {
      */
     public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
 
-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase();
+        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
         return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableCell.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
@@ -7,11 +7,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/AtomicTextBlock.java

@@ -1,8 +1,9 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -10,12 +11,11 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.AtomicTextBlockData;
-import com.knecon.fforesight.service.layoutparser.processor.factory.SearchTextWithTextPositionDto;
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 
 import lombok.AccessLevel;
@@ -38,11 +38,20 @@ public class AtomicTextBlock implements TextBlock {
     //string coordinates
     Boundary boundary;
     String searchText;
-    List<Integer> lineBreaks;
+    @Builder.Default
+    List<Integer> lineBreaks = new ArrayList<>();
+    @Builder.Default
+    List<Boundary> boldTextBoundaries = new ArrayList<>();
+    @Builder.Default
+    List<Boundary> italicTextBoundaries = new ArrayList<>();
+    String orientation;
+    int textDirection;
 
     //position coordinates
-    List<Integer> stringIdxToPositionIdx;
-    List<Rectangle2D> positions;
+    @Builder.Default
+    List<Integer> stringIdxToPositionIdx = new ArrayList<>();
+    @Builder.Default
+    List<Rectangle2D> positions = new ArrayList<>();
 
     @EqualsAndHashCode.Exclude
     SemanticNode parent;
@@ -55,23 +64,34 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    public static AtomicTextBlock fromSearchTextWithTextPositionDto(SearchTextWithTextPositionDto searchTextWithTextPositionDto,
-                                                                    SemanticNode parent,
-                                                                    int stringOffset,
-                                                                    Long textBlockIdx,
-                                                                    Integer numberOnPage,
-                                                                    Page page) {
+    public static AtomicTextBlock fromSearchTextWithTextPosition(String searchText,
+                                                                 List<Integer> lineBreaks,
+                                                                 List<Boundary> boldTextBoundaries,
+                                                                 List<Boundary> italicTextBoundaries,
+                                                                 List<Rectangle2D> positions,
+                                                                 List<Integer> stringIdxToPositionIdx,
+                                                                 long idx,
+                                                                 SemanticNode parent,
+                                                                 int numberOnPage,
+                                                                 Page page,
+                                                                 int offset,
+                                                                 String orientation,
+                                                                 int textDirection) {
 
         return AtomicTextBlock.builder()
-                .id(textBlockIdx)
+                .id(idx)
                 .parent(parent)
-                .searchText(searchTextWithTextPositionDto.getSearchText())
+                .searchText(searchText)
                 .numberOnPage(numberOnPage)
                 .page(page)
-                .lineBreaks(searchTextWithTextPositionDto.getLineBreaks())
-                .positions(searchTextWithTextPositionDto.getPositions())
-                .stringIdxToPositionIdx(searchTextWithTextPositionDto.getStringCoordsToPositionCoords())
-                .boundary(new Boundary(stringOffset, stringOffset + searchTextWithTextPositionDto.getSearchText().length()))
+                .lineBreaks(lineBreaks)
+                .boldTextBoundaries(boldTextBoundaries)
+                .italicTextBoundaries(italicTextBoundaries)
+                .positions(positions)
+                .stringIdxToPositionIdx(stringIdxToPositionIdx)
+                .boundary(new Boundary(offset, offset + searchText.length()))
+                .textDirection(textDirection)
+                .orientation(orientation)
                 .build();
     }
 
@@ -82,30 +102,27 @@ public class AtomicTextBlock implements TextBlock {
                 .id(textBlockIdx)
                 .boundary(new Boundary(stringOffset, stringOffset))
                 .searchText("")
-                .lineBreaks(Collections.emptyList())
                 .page(page)
                 .numberOnPage(numberOnPage)
-                .stringIdxToPositionIdx(Collections.emptyList())
-                .positions(Collections.emptyList())
                 .parent(parent)
                 .build();
     }
 
 
-    public static AtomicTextBlock fromAtomicTextBlockData(AtomicTextBlockData atomicTextBlockData,
-                                                          AtomicPositionBlockData atomicPositionBlockData,
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData,
+                                                          DocumentPositionData documentPositionData,
                                                           SemanticNode parent,
                                                           Page page) {
 
         return AtomicTextBlock.builder()
-                .id(atomicTextBlockData.getId())
-                .numberOnPage(atomicTextBlockData.getNumberOnPage())
+                .id(documentTextData.getId())
+                .numberOnPage(documentTextData.getNumberOnPage())
                 .page(page)
-                .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
-                .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList())
-                .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList())
-                .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
+                .boundary(new Boundary(documentTextData.getStart(), documentTextData.getEnd()))
+                .searchText(documentTextData.getSearchText())
+                .lineBreaks(Arrays.stream(documentTextData.getLineBreaks()).boxed().toList())
+                .stringIdxToPositionIdx(Arrays.stream(documentPositionData.getStringIdxToPositionIdx()).boxed().toList())
+                .positions(toRectangle2DList(documentPositionData.getPositions()))
                 .parent(parent)
                 .build();
     }
@@ -191,7 +208,7 @@ public class AtomicTextBlock implements TextBlock {
         List<Rectangle2D> rectanglesPerLine = stringBoundary.split(getAllLineBreaksInBoundary(stringBoundary))
                 .stream()
                 .map(this::getPositions)
-                .map(RectangleTransformations::rectangleUnionWithGaps)
+                .map(RectangleTransformations::rectangleBBoxWithGaps)
                 .flatMap(Collection::stream)
                 .toList();
         Map<Page, List<Rectangle2D>> rectanglePerLinePerPage = new HashMap<>();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/ConcatenatedTextBlock.java

@@ -1,8 +1,9 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -10,8 +11,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 
 import lombok.AccessLevel;
 import lombok.Data;
@@ -182,4 +183,38 @@ public class ConcatenatedTextBlock implements TextBlock {
         return getSearchText();
     }
 
+
+    @Override
+    public List<Boundary> getBoldTextBoundaries() {
+
+        return getAtomicTextBlocks().stream().map(AtomicTextBlock::getBoldTextBoundaries).flatMap(Collection::stream).toList();
+    }
+
+
+    @Override
+    public List<Boundary> getItalicTextBoundaries() {
+
+        return getAtomicTextBlocks().stream().map(AtomicTextBlock::getItalicTextBoundaries).flatMap(Collection::stream).toList();
+    }
+
+
+    @Override
+    public String getOrientation() {
+
+        if (atomicTextBlocks.isEmpty()) {
+            return "";
+        }
+        return atomicTextBlocks.get(0).getOrientation();
+    }
+
+
+    @Override
+    public int getTextDirection() {
+
+        if (atomicTextBlocks.isEmpty()) {
+            return 0;
+        }
+        return atomicTextBlocks.get(0).getTextDirection();
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/TextBlock.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
 import static java.lang.String.format;
 
@@ -10,8 +10,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 
 public interface TextBlock extends CharSequence {
 
@@ -21,6 +21,18 @@ public interface TextBlock extends CharSequence {
     List<AtomicTextBlock> getAtomicTextBlocks();
 
 
+    List<Boundary> getBoldTextBoundaries();
+
+
+    List<Boundary> getItalicTextBoundaries();
+
+
+    String getOrientation();
+
+
+    int getTextDirection();
+
+
     Boundary getBoundary();
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/textblock/TextBlockCollector.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
 
 import java.util.Set;
 import java.util.function.BiConsumer;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/image/ClassifiedImage.java

@@ -1,8 +1,8 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.image;
+package com.knecon.fforesight.service.layoutparser.processor.model.image;
 
 import java.awt.geom.Rectangle2D;
 
-import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
 
 import lombok.Data;
 import lombok.NonNull;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Cell.java

@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.model.table;
 
 import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.classification.utils.TextNormalizationUtilities;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/CellPosition.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.model.table;
 
 import lombok.RequiredArgsConstructor;
 import lombok.Value;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/CleanRulings.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.model.table;
 
 import java.util.List;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Rectangle.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.model.table;
 
 import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Ruling.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.model.table;
 
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
@@ -11,8 +11,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.utils.CohenSutherlandClipping;
-import com.knecon.fforesight.service.layoutparser.processor.classification.utils.DoubleComparisons;
+import com.knecon.fforesight.service.layoutparser.processor.utils.CohenSutherlandClipping;
+import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -29,6 +29,24 @@ public class Ruling extends Line2D.Float {
     }
 
 
+    public Ruling straightenVertical() {
+
+        double y1 = Math.min(getY1(), getY2());
+        double y2 = Math.max(getY1(), getY2());
+        double x = (getX1() + getX2()) / 2;
+        return new Ruling(new Point2D.Double(x, y1), new Point2D.Double(x, y2));
+    }
+
+
+    public Ruling straightenHorizontal() {
+
+        double x1 = Math.min(getX1(), getX2());
+        double x2 = Math.max(getX1(), getX2());
+        double y = (getY1() + getY2()) / 2;
+        return new Ruling(new Point2D.Double(x1, y), new Point2D.Double(x2, y));
+    }
+
+
     public static List<Ruling> cropRulingsToArea(List<Ruling> rulings, Rectangle2D area) {
 
         ArrayList<Ruling> rv = new ArrayList<>();
@@ -428,6 +446,16 @@ public class Ruling extends Line2D.Float {
     }
 
 
+    public boolean almostMatches(Ruling ruling) {
+
+        final float TOLERANCE = 1;
+        return Math.abs(ruling.getX1() - x1) < TOLERANCE &&//
+                Math.abs(ruling.getY1() - y1) < TOLERANCE &&//
+                Math.abs(ruling.getX2() - x2) < TOLERANCE &&//
+                Math.abs(ruling.getY2() - y2) < TOLERANCE;
+    }
+
+
     private enum SOType {
         VERTICAL,
         HRIGHT,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/TablePageBlock.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.model.table;
 
 import java.awt.geom.Point2D;
 import java.util.ArrayList;
@@ -8,9 +8,9 @@ import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
 
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.classification.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import lombok.Getter;
 import lombok.Setter;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.text;
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import org.apache.pdfbox.text.TextPosition;
 import org.springframework.beans.BeanUtils;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/SearchableText.java

@@ -1,8 +1,8 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.text;
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import java.util.ArrayList;
 import java.util.List;
-import com.knecon.fforesight.service.layoutparser.processor.classification.utils.TextNormalizationUtilities;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Getter;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/StringFrequencyCounter.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.text;
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import java.util.HashMap;
 import java.util.Map;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextDirection.java

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.classification.model.text;
+package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;