try native build

Merge branch 'fixGradleDeploy' into 'main'
Fix gradle deploy See merge request fforesight/layout-parser!3
2023-07-27 13:50:32 +02:00 · 2023-07-27 13:42:10 +02:00 · 2023-07-27 13:29:14 +02:00 · 2023-07-27 13:21:01 +02:00 · 2023-07-27 13:00:26 +02:00 · 2023-07-27 12:32:10 +02:00
158 changed files with 57459 additions and 741 deletions
@@ -18,6 +18,7 @@ target/
 .settings
 .springBeans
 .sts4-cache
+.gradle

 ### IntelliJ IDEA ###
 .idea
@@ -1,4 +1,4 @@
 include:
  - project: 'gitlab/gitlab'
    ref: 'main'
-    file: 'ci-templates/maven_java.yml'
+    file: 'ci-templates/gradle_java.yml'
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.7/apache-maven-3.8.7-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar
@@ -0,0 +1,7 @@
+plugins {
+    `kotlin-dsl`
+}
+
+repositories {
+    gradlePluginPortal()
+}
@@ -0,0 +1,56 @@
+plugins {
+    java
+    pmd
+    checkstyle
+    jacoco
+}
+
+group = "com.knecon.fforesight"
+version = "0.1-SNAPSHOT"
+
+java.sourceCompatibility = JavaVersion.VERSION_17
+java.targetCompatibility = JavaVersion.VERSION_17
+
+tasks.jacocoTestReport {
+    reports {
+        xml.required.set(false)
+        csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
+    }
+}
+
+tasks.pmdMain {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
+}
+
+tasks.pmdTest {
+    pmd.ruleSetFiles = files("${rootDir}/config/pmd/test_pmd.xml")
+}
+
+tasks.named<Test>("test") {
+    useJUnitPlatform()
+}
+
+tasks.test {
+    finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
+}
+
+tasks.jacocoTestReport {
+    dependsOn(tasks.test) // tests are required to run before generating the report
+    reports {
+        xml.required.set(true)
+        csv.required.set(false)
+    }
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven {
+        url = uri("https://nexus.knecon.com/repository/gindev/");
+        credentials {
+            username = providers.gradleProperty("mavenUser").getOrNull();
+            password = providers.gradleProperty("mavenPassword").getOrNull();
+        }
+    }
+}
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <property
+            name="severity"
+            value="error"/>
+    <module name="TreeWalker">
+        <module name="SuppressWarningsHolder"/>
+        <module name="MissingDeprecated"/>
+        <module name="MissingOverride"/>
+        <module name="AnnotationLocation"/>
+        <module name="JavadocStyle"/>
+        <module name="NonEmptyAtclauseDescription"/>
+        <module name="IllegalImport"/>
+        <module name="RedundantImport"/>
+        <module name="RedundantModifier"/>
+        <module name="EmptyBlock"/>
+        <module name="DefaultComesLast"/>
+        <module name="EmptyStatement"/>
+        <module name="EqualsHashCode"/>
+        <module name="ExplicitInitialization"/>
+        <module name="IllegalInstantiation"/>
+        <module name="ModifiedControlVariable"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="PackageDeclaration"/>
+        <module name="ParameterAssignment"/>
+        <module name="SimplifyBooleanExpression"/>
+        <module name="SimplifyBooleanReturn"/>
+        <module name="StringLiteralEquality"/>
+        <module name="OneStatementPerLine"/>
+        <module name="FinalClass"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="UpperEll"/>
+        <module name="OuterTypeFilename"/>
+    </module>
+    <module name="FileTabCharacter"/>
+    <module name="SuppressWarningsFilter"/>
+</module>
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon ruleset checks the code for bad stuff
+    </description>
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="BeanMembersShouldSerialize"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+    </rule>
+
+</ruleset>
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<ruleset name="Custom ruleset"
+         xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
+
+    <description>
+        Knecon test ruleset checks the code for bad stuff
+    </description>
+
+
+    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
+        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
+        <exclude name="BeanMembersShouldSerialize"/>
+        <exclude name="AvoidLiteralsInIfCondition"/>
+        <exclude name="AvoidDuplicateLiterals"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="AvoidFieldNameMatchingTypeName"/>
+    </rule>
+
+</ruleset>
@@ -0,0 +1 @@
+version = 0.1.0
@@ -1,99 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>platform-docker-dependency</artifactId>
-        <version>0.1.0</version>
-        <relativePath/>
-    </parent>
-
-    <groupId>com.knecon.fforesight</groupId>
-    <artifactId>layoutparser-service-image</artifactId>
-    <version>0.1-SNAPSHOT</version>
-    <packaging>pom</packaging>
-
-    <properties>
-        <service.server>layoutparser-service-server</service.server>
-        <platform.jar>${service.server}.jar</platform.jar>
-        <docker.skip.push>false</docker.skip.push>
-        <docker.image.prefix>ff</docker.image.prefix>
-        <docker.image.name>${docker.image.prefix}/${service.server}</docker.image.name>
-    </properties>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-dependency-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-resources-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-            </plugin>
-            <plugin>
-                <groupId>io.fabric8</groupId>
-                <artifactId>docker-maven-plugin</artifactId>
-            </plugin>
-        </plugins>
-
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-dependency-plugin</artifactId>
-                    <executions>
-                        <execution>
-                            <id>download-platform-jar</id>
-                            <phase>prepare-package</phase>
-                            <goals>
-                                <goal>copy</goal>
-                            </goals>
-                            <configuration>
-                                <artifactItems>
-                                    <dependency>
-                                        <groupId>${project.groupId}</groupId>
-                                        <artifactId>${service.server}</artifactId>
-                                        <version>${project.version}</version>
-                                        <type>jar</type>
-                                        <overWrite>true</overWrite>
-                                        <destFileName>${platform.jar}</destFileName>
-                                    </dependency>
-                                </artifactItems>
-                                <outputDirectory>${docker.build.directory}</outputDirectory>
-                            </configuration>
-                        </execution>
-                    </executions>
-                </plugin>
-                <plugin>
-                    <groupId>io.fabric8</groupId>
-                    <artifactId>docker-maven-plugin</artifactId>
-                    <configuration>
-                        <images>
-                            <image>
-                                <name>${docker.image.name}</name>
-                                <build>
-                                    <dockerFileDir>${docker.build.directory}</dockerFileDir>
-                                    <args>
-                                        <PLATFORM_JAR>${platform.jar}</PLATFORM_JAR>
-                                    </args>
-                                    <tags>
-                                        <tag>${docker.image.version}</tag>
-                                        <tag>latest</tag>
-                                    </tags>
-                                </build>
-                            </image>
-                        </images>
-                    </configuration>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
-</project>
@@ -1,9 +0,0 @@
-FROM red/base-image:2.0.2
-
-ARG PLATFORM_JAR
-
-ENV PLATFORM_JAR ${PLATFORM_JAR}
-
-ENV USES_ELASTICSEARCH false
-
-COPY ["${PLATFORM_JAR}", "/"]
@@ -0,0 +1,6 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.1.0"
+}
+
+description = "layoutparser-service-internal-api"
@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>0.1-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>layoutparser-service-internal-api</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>${guava.version}</version>
-        </dependency>
-    </dependencies>
-
-</project>
@@ -12,10 +12,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DocumentData {

-    PageData[] pages;
-    AtomicTextBlockData[] atomicTextBlocks;
-    AtomicPositionBlockData[] atomicPositionBlocks;
-    DocumentTreeData documentTreeData;
-
+    DocumentPage[] documentPages;
+    DocumentTextData[] documentTextData;
+    DocumentPositionData[] documentPositions;
+    DocumentStructure documentStructure;

 }
@@ -4,13 +4,15 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;

@Data
@Builder
+@NoArgsConstructor
@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class PageData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentPage {

    int number;
    int height;
@@ -4,13 +4,15 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;

@Data
@Builder
+@NoArgsConstructor
@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class AtomicPositionBlockData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentPositionData {

    Long id;
    int[] stringIdxToPositionIdx;
@@ -8,16 +8,15 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;

@Data
@Builder
-@AllArgsConstructor
@NoArgsConstructor
+@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentTreeData {
+public class DocumentStructure {

    EntryData root;

@@ -37,7 +36,7 @@ public class DocumentTreeData {

    public Stream<EntryData> streamAllEntries() {

-        return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentTreeData::flatten);
+        return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentStructure::flatten);
    }


@@ -49,14 +48,15 @@ public class DocumentTreeData {

    private static Stream<EntryData> flatten(EntryData entry) {

-        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTreeData::flatten));
+        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentStructure::flatten));
    }


+    @Data
    @Builder
-    @Getter
+    @NoArgsConstructor
    @AllArgsConstructor
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    @FieldDefaults(level = AccessLevel.PRIVATE)
    public static class EntryData {

        NodeType type;
@@ -6,13 +6,15 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;

@Data
@Builder
+@NoArgsConstructor
@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class AtomicTextBlockData {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentTextData {

    Long id;
    Long page;
@@ -22,6 +24,4 @@ public class AtomicTextBlockData {
    int end;
    int[] lineBreaks;

-
-
 }
@@ -12,4 +12,5 @@ public class TableData {
    List<RowData> rowData;
    Integer numberOfCols;
    Integer numberOfRows;
+
 }
@@ -0,0 +1,21 @@
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("io.freefair.lombok") version "8.1.0"
+}
+
+dependencies {
+    implementation(project(":layoutparser-service-internal-api"))
+
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+
+    implementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
+    implementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
+}
+
+description = "layoutparser-service-processor"
@@ -1,71 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>0.1-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>layoutparser-service-processor</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-shared-api-v1</artifactId>
-            <version>2.36.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>tenant-commons</artifactId>
-            <version>${tennat-commons.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>layoutparser-service-internal-api</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.iqser.red.commons</groupId>
-            <artifactId>storage-commons</artifactId>
-            <version>${storage-commons.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.pdfbox</groupId>
-            <artifactId>pdfbox-tools</artifactId>
-            <version>${pdfbox.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-afterburner</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-jsr310</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-web</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-amqp</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>RELEASE</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-</project>
@@ -48,40 +48,40 @@ public class LayoutParsingPipeline {
    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {

        long start = System.currentTimeMillis();
-        PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());

-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
+        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
+            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
+            }
+
+            TableServiceResponse tableServiceResponse = new TableServiceResponse();
+            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
+            }
+
+            Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
+            int numberOfPages = originDocument.getNumberOfPages();
+
+            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
+            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+
+            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+            }
+
+            return LayoutParsingFinishedEvent.builder()
+                    .identifier(layoutParsingRequest.identifier())
+                    .numberOfPages(numberOfPages)
+                    .duration(System.currentTimeMillis() - start)
+                    .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
+                            layoutParsingRequest.structureFileStorageId(),
+                            layoutParsingRequest.textBlockFileStorageId(),
+                            layoutParsingRequest.positionBlockFileStorageId(),
+                            layoutParsingRequest.pageFileStorageId()))
+                    .build();
        }
-
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
-        }
-
-        Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
-        int numberOfPages = originDocument.getNumberOfPages();
-        originDocument.close();
-
-        layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-
-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
-            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
-            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
-        }
-
-        return LayoutParsingFinishedEvent.builder()
-                .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(numberOfPages)
-                .duration(System.currentTimeMillis() - start)
-                .message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId()))
-                .build();
    }


@@ -102,6 +102,7 @@ public class LayoutParsingPipeline {
        }

        sectionsBuilderService.buildSections(classificationDocument);
+        sectionsBuilderService.addImagesToSections(classificationDocument);

        return DocumentGraphFactory.buildDocumentGraph(classificationDocument);
    }
@@ -114,7 +115,8 @@ public class LayoutParsingPipeline {

        long start = System.currentTimeMillis();

-        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType, originDocument,
+        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
+                originDocument,
                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));

@@ -15,11 +15,11 @@ import org.springframework.stereotype.Service;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.AtomicTextBlockData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTreeData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.PageData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
@@ -71,10 +71,10 @@ public class LayoutParsingStorageService {

    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentTreeData());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getAtomicTextBlocks());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getAtomicPositionBlocks());
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getPages());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.textBlockFileStorageId(), documentData.getDocumentTextData());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.positionBlockFileStorageId(), documentData.getDocumentPositions());
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), documentData.getDocumentPages());
    }


@@ -92,20 +92,20 @@ public class LayoutParsingStorageService {

    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {

-        PageData[] pageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), PageData[].class);
-        AtomicTextBlockData[] atomicTextBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
+        DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class);
+        DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
                layoutParsingRequest.textBlockFileStorageId(),
-                AtomicTextBlockData[].class);
-        AtomicPositionBlockData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
+                DocumentTextData[].class);
+        DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
                layoutParsingRequest.positionBlockFileStorageId(),
-                AtomicPositionBlockData[].class);
-        DocumentTreeData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentTreeData.class);
+                DocumentPositionData[].class);
+        DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class);

        return DocumentData.builder()
-                .documentTreeData(tableOfContentsData)
-                .atomicPositionBlocks(atomicPositionBlockData)
-                .atomicTextBlocks(atomicTextBlockData)
-                .pages(pageData)
+                .documentStructure(tableOfContentsData)
+                .documentPositions(atomicPositionBlockData)
+                .documentTextData(documentTextDataBlockData)
+                .documentPages(documentPageData)
                .build();
    }

@@ -5,6 +5,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;

 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
@@ -132,7 +133,7 @@ public class SearchTextWithTextPositionFactory {

    private static void addTextPositionWithFontType(RedTextPosition currentTextPosition, String fontType, List<Integer> fontTypePositions, int stringIdx) {

-        if (currentTextPosition.getFontName().toLowerCase().contains(fontType)) {
+        if (currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
            fontTypePositions.add(stringIdx);
        }
    }
@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

+import java.util.Locale;
+
 public enum ImageType {
    LOGO,
    FORMULA,
@@ -10,7 +12,7 @@ public enum ImageType {

    public static ImageType fromString(String imageType) {

-        return switch (imageType.toLowerCase()) {
+        return switch (imageType.toLowerCase(Locale.ROOT)) {
            case "logo" -> ImageType.LOGO;
            case "formula" -> ImageType.FORMULA;
            case "signature" -> ImageType.SIGNATURE;
@@ -6,6 +6,7 @@ import java.awt.geom.Rectangle2D;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -285,7 +286,7 @@ public interface SemanticNode {
     */
    default boolean containsStringIgnoreCase(String string) {

-        return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
+        return getTextBlock().getSearchText().toLowerCase(Locale.ROOT).contains(string.toLowerCase(Locale.ROOT));
    }


@@ -5,6 +5,7 @@ import static java.lang.String.format;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -67,7 +68,7 @@ public class Table implements SemanticNode {
     */
    public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {

-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase();
+        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
    }

@@ -11,8 +11,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.AtomicTextBlockData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
@@ -109,20 +109,20 @@ public class AtomicTextBlock implements TextBlock {
    }


-    public static AtomicTextBlock fromAtomicTextBlockData(AtomicTextBlockData atomicTextBlockData,
-                                                          AtomicPositionBlockData atomicPositionBlockData,
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData,
+                                                          DocumentPositionData documentPositionData,
                                                          SemanticNode parent,
                                                          Page page) {

        return AtomicTextBlock.builder()
-                .id(atomicTextBlockData.getId())
-                .numberOnPage(atomicTextBlockData.getNumberOnPage())
+                .id(documentTextData.getId())
+                .numberOnPage(documentTextData.getNumberOnPage())
                .page(page)
-                .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
-                .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList())
-                .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList())
-                .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
+                .boundary(new Boundary(documentTextData.getStart(), documentTextData.getEnd()))
+                .searchText(documentTextData.getSearchText())
+                .lineBreaks(Arrays.stream(documentTextData.getLineBreaks()).boxed().toList())
+                .stringIdxToPositionIdx(Arrays.stream(documentPositionData.getStringIdxToPositionIdx()).boxed().toList())
+                .positions(toRectangle2DList(documentPositionData.getPositions()))
                .parent(parent)
                .build();
    }
@@ -4,12 +4,14 @@ import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.AtomicTextBlockData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.PageData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTreeData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
@@ -26,36 +28,38 @@ public class DocumentDataMapper {

    public DocumentData toDocumentData(Document document) {

-        List<AtomicTextBlockData> atomicTextBlockData = document.streamTerminalTextBlocksInOrder()
+        List<DocumentTextData> documentTextData = document.streamTerminalTextBlocksInOrder()
                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
                .distinct()
                .map(DocumentDataMapper::toAtomicTextBlockData)
                .toList();

-        List<AtomicPositionBlockData> atomicPositionBlockData = document.streamTerminalTextBlocksInOrder()
+        List<DocumentPositionData> atomicPositionBlockData = document.streamTerminalTextBlocksInOrder()
                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
                .distinct()
                .map(DocumentDataMapper::toAtomicPositionBlockData)
                .toList();

-        List<PageData> pageData = document.getPages().stream().map(DocumentDataMapper::toPageData).toList();
-        DocumentTreeData tableOfContentsData = toDocumentTreeData(document.getDocumentTree());
+        Set<Long> nonEmptyTextBlocks = documentTextData.stream().mapToLong(DocumentTextData::getId).boxed().collect(Collectors.toSet());
+
+        List<DocumentPage> documentPageData = document.getPages().stream().map(DocumentDataMapper::toPageData).toList();
+        DocumentStructure tableOfContentsData = toDocumentTreeData(document.getDocumentTree());
        return DocumentData.builder()
-                .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
-                .atomicPositionBlocks(atomicPositionBlockData.toArray(new AtomicPositionBlockData[0]))
-                .pages(pageData.toArray(new PageData[0]))
-                .documentTreeData(tableOfContentsData)
+                .documentTextData(documentTextData.toArray(new DocumentTextData[0]))
+                .documentPositions(atomicPositionBlockData.toArray(new DocumentPositionData[0]))
+                .documentPages(documentPageData.toArray(new DocumentPage[0]))
+                .documentStructure(tableOfContentsData)
                .build();
    }


-    private DocumentTreeData toDocumentTreeData(DocumentTree documentTree) {
+    private DocumentStructure toDocumentTreeData(DocumentTree documentTree) {

-        return new DocumentTreeData(toEntryData(documentTree.getRoot()));
+        return new DocumentStructure(toEntryData(documentTree.getRoot()));
    }


-    private DocumentTreeData.EntryData toEntryData(DocumentTree.Entry entry) {
+    private DocumentStructure.EntryData toEntryData(DocumentTree.Entry entry) {

        Long[] atomicTextBlocks;

@@ -72,7 +76,7 @@ public class DocumentDataMapper {
            default -> new HashMap<>();
        };

-        return DocumentTreeData.EntryData.builder()
+        return DocumentStructure.EntryData.builder()
                .treeId(toPrimitiveIntArray(entry.getTreeId()))
                .children(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
                .type(entry.getType())
@@ -89,15 +93,15 @@ public class DocumentDataMapper {
    }


-    private PageData toPageData(Page p) {
+    private DocumentPage toPageData(Page p) {

-        return PageData.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
+        return DocumentPage.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
    }


-    private AtomicTextBlockData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {
+    private DocumentTextData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {

-        return AtomicTextBlockData.builder()
+        return DocumentTextData.builder()
                .id(atomicTextBlock.getId())
                .page(atomicTextBlock.getPage().getNumber().longValue())
                .searchText(atomicTextBlock.getSearchText())
@@ -109,9 +113,9 @@ public class DocumentDataMapper {
    }


-    private AtomicPositionBlockData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {
+    private DocumentPositionData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {

-        return AtomicPositionBlockData.builder()
+        return DocumentPositionData.builder()
                .id(atomicTextBlock.getId())
                .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
@@ -7,11 +7,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.AtomicPositionBlockData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.AtomicTextBlockData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTreeData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.PageData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Footer;
@@ -39,23 +39,23 @@ public class DocumentGraphMapper {
        DocumentTree documentTree = new DocumentTree(document);
        Context context = new Context(documentData, documentTree);

-        context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList());
+        context.pages.addAll(Arrays.stream(documentData.getDocumentPages()).map(DocumentGraphMapper::buildPage).toList());

-        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentTreeData().getRoot().getChildren(), context));
+        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));

        document.setDocumentTree(context.documentTree);
        document.setPages(new HashSet<>(context.pages));
-        document.setNumberOfPages(documentData.getPages().length);
+        document.setNumberOfPages(documentData.getDocumentPages().length);

        document.setTextBlock(document.getTextBlock());
        return document;
    }


-    private List<DocumentTree.Entry> buildEntries(List<DocumentTreeData.EntryData> entries, Context context) {
+    private List<DocumentTree.Entry> buildEntries(List<DocumentStructure.EntryData> entries, Context context) {

        List<DocumentTree.Entry> newEntries = new LinkedList<>();
-        for (DocumentTreeData.EntryData entryData : entries) {
+        for (DocumentStructure.EntryData entryData : entries) {

            List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();

@@ -154,14 +154,14 @@ public class DocumentGraphMapper {

    private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {

-        return AtomicTextBlock.fromAtomicTextBlockData(context.atomicTextBlockData.get(Math.toIntExact(atomicTextBlockId)),
+        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)),
                context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
                parent,
-                getPage(context.atomicTextBlockData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                getPage(context.documentTextDataBlockData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
    }


-    private Page buildPage(PageData p) {
+    private Page buildPage(DocumentPage p) {

        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
    }
@@ -180,16 +180,16 @@ public class DocumentGraphMapper {

        private final DocumentTree documentTree;
        private final List<Page> pages;
-        private final List<AtomicTextBlockData> atomicTextBlockData;
-        private final List<AtomicPositionBlockData> atomicPositionBlockData;
+        private final List<DocumentTextData> documentTextDataBlockData;
+        private final List<DocumentPositionData> atomicPositionBlockData;


        Context(DocumentData documentData, DocumentTree documentTree) {

            this.documentTree = documentTree;
            this.pages = new LinkedList<>();
-            this.atomicTextBlockData = Arrays.stream(documentData.getAtomicTextBlocks()).toList();
-            this.atomicPositionBlockData = Arrays.stream(documentData.getAtomicPositionBlocks()).toList();
+            this.documentTextDataBlockData = Arrays.stream(documentData.getDocumentTextData()).toList();
+            this.atomicPositionBlockData = Arrays.stream(documentData.getDocumentPositions()).toList();

        }

@@ -28,6 +28,22 @@ public class Ruling extends Line2D.Float {
        super(p1, p2);
    }

+    public Ruling straightenVertical() {
+
+        double y1 = Math.min(getY1(), getY2());
+        double y2 = Math.max(getY1(), getY2());
+        double x = (getX1() + getX2()) / 2;
+        return new Ruling(new Point2D.Double(x, y1), new Point2D.Double(x, y2));
+    }
+
+    public Ruling straightenHorizontal() {
+
+        double x1 = Math.min(getX1(), getX2());
+        double x2 = Math.max(getX1(), getX2());
+        double y = (getY1() + getY2()) / 2;
+        return new Ruling(new Point2D.Double(x1, y), new Point2D.Double(x2, y));
+    }
+

    public static List<Ruling> cropRulingsToArea(List<Ruling> rulings, Rectangle2D area) {

@@ -4,6 +4,7 @@ import java.awt.geom.AffineTransform;
 import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;
@@ -76,7 +77,7 @@ public class TextPositionSequence implements CharSequence {

        RedTextPosition textPosition = textPositionAt(index);
        String text = textPosition.getUnicode();
-        return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
+        return caseInSensitive ? text.toLowerCase(Locale.ROOT).charAt(0) : text.charAt(0);
    }


@@ -223,7 +224,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public String getFont() {

-        return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
+        return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
    }


@@ -231,7 +232,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public String getFontStyle() {

-        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();
+        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);

        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
            return "bold, italic";
@@ -6,6 +6,7 @@ import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
@@ -17,7 +18,7 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
@Service
 public class BodyTextFrameService {

-    private static final float APPROXIMATE_HEADER_LINE_COUNT = 2.0f;
+


    /**
@@ -68,7 +69,14 @@ public class BodyTextFrameService {
     * @param landscape               Calculate for landscape or portrait
     * @return Rectangle of the text frame
     */
-    public Rectangle calculateBodyTextFrame(List<ClassificationPage> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
+    public Rectangle calculateBodyTextFrame(List<ClassificationPage> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape, LayoutParsingType layoutParsingType) {
+
+        float approximateHeaderLineCount;
+        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
+            approximateHeaderLineCount = 3.3f;
+        } else {
+            approximateHeaderLineCount = 2.9f;
+        }

        BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();

@@ -87,7 +95,8 @@ public class BodyTextFrameService {
                    }

                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (approxLineCount < APPROXIMATE_HEADER_LINE_COUNT) {
+                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10)
+                            || !layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount){
                        continue;
                    }

@@ -6,6 +6,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
 import java.util.stream.Stream;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;

@@ -93,7 +94,7 @@ public class GapsAcrossLinesService {
        int lineCount = 1;


-        public GapAcrossLines(Rectangle2D rectangle2D) {
+        GapAcrossLines(Rectangle2D rectangle2D) {

            this.rectangle2D = correctRectangle(rectangle2D);
        }
@@ -108,7 +108,7 @@ public class PdfParsingService {
            imageServiceResponseAdapter.findOcr(classificationPage);
        }

-        tableExtractionService.extractTables(cleanRulings, classificationPage);
+        tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
        buildPageStatistics(classificationPage);
        increaseDocumentStatistics(classificationPage, document);

@@ -53,7 +53,10 @@ public class RulingCleaningService {
        }
        List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);

-        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
+        return CleanRulings.builder()
+                .vertical(verticalRulingLines)
+                .horizontal(horizontalRulingLines)
+                .build();
    }


@@ -12,6 +12,7 @@ import java.util.Set;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -78,9 +79,9 @@ public class TableExtractionService {
     * @param cleanRulings The lines used to build the table.
     * @param page         Page object that contains textblocks and statistics.
     */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {
+    public void extractTables(CleanRulings cleanRulings, ClassificationPage page, LayoutParsingType layoutParsingType) {

-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
+        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical(), layoutParsingType);

        List<TextPageBlock> toBeRemoved = new ArrayList<>();

@@ -134,13 +135,16 @@ public class TableExtractionService {
    }


-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, LayoutParsingType layoutParsingType) {

-        for (Ruling r : horizontalRulingLines) {
-            if (r.getX2() < r.getX1()) {
-                double a = r.getX2();
-                r.x2 = (float) r.getX1();
-                r.x1 = (float) a;
+        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
+            // TODO: breaks some tables, for example "1 Abamectin Prr.pdf" try to fix this upstream in RulingCleaningService
+            for (Ruling r : horizontalRulingLines) {
+                if (r.getX2() < r.getX1()) {
+                    double a = r.getX2();
+                    r.x2 = (float) r.getX1();
+                    r.x1 = (float) a;
+                }
            }
        }

@@ -28,31 +28,30 @@ public class TextPositionSequenceSorter {
        List<PageContents> textPositionSequencesPerPage = new LinkedList<>();
        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {

-            PDDocument pdDocument = Loader.loadPDF(inputStream);
+            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {

-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {

-                PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-                PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-                stripper.setPageNumber(pageNumber);
-                stripper.setSortByPosition(true);
-                stripper.setStartPage(pageNumber);
-                stripper.setEndPage(pageNumber);
-                stripper.setPdpage(pdPage);
-                stripper.getText(pdDocument);
+                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+                    stripper.setPageNumber(pageNumber);
+                    stripper.setSortByPosition(true);
+                    stripper.setStartPage(pageNumber);
+                    stripper.setEndPage(pageNumber);
+                    stripper.setPdpage(pdPage);
+                    stripper.getText(pdDocument);

-                Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
-                        .stream()
-                        .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
+                    Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
+                            .stream()
+                            .collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));

-                var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());
+                    var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());

-                textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
-                        RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
-                        RectangleTransformations.toRectangle2D(pdPage.getMediaBox())));
+                    textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
+                            RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
+                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox())));
+                }
            }
-
-            pdDocument.close();
        }

        return textPositionSequencesPerPage;
@@ -39,7 +39,11 @@ public class DocuMineBlockificationService {
        List<TextPositionSequence> chunkWords = new ArrayList<>();
        List<AbstractPageBlock> chunkBlockList1 = new ArrayList<>();

-        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
+        float minX = 1000;
+        float maxX = 0;
+        float minY = 1000;
+        float maxY = 0;
+
        TextPositionSequence prev = null;

        boolean wasSplitted = false;
@@ -8,6 +8,7 @@ import java.util.regex.Pattern;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -33,8 +34,8 @@ public class DocuMineClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true);
+        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.DOCUMINE);
+        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.DOCUMINE);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
@@ -69,20 +70,7 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (textBlock.getText().length() > 6 && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter()
-                .getMostPopular() || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()) && PositionUtils.getApproxLineCount(textBlock) < 5.9
-
-                && (textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && !matcher2.matches() && !textBlock.toString()
-                .contains(":") || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && !matcher2.matches() && !textBlock.toString()
-                .contains(":") || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("FIGURE") || textBlock.toString()
-                .startsWith("TABLE")) && !textBlock.toString().endsWith(":")) {
-            textBlock.setClassification(PageBlockType.getHeadlineType(1));
-            document.setHeadlines(true);
-
-        } else if (matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher3.matches() && !matcher2.matches()) {
-            textBlock.setClassification(PageBlockType.getHeadlineType(2));
-            document.setHeadlines(true);
-        } else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

@@ -95,6 +83,19 @@ public class DocuMineClassificationService {
            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                textBlock.setClassification(PageBlockType.TITLE);
            }
+        } else if (textBlock.getText().length() > 5 && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter()
+                .getMostPopular() || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()) && PositionUtils.getApproxLineCount(textBlock) < 5.9
+
+                && (textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && !matcher2.matches() && !textBlock.toString()
+                .contains(":") || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && !matcher2.matches() && !textBlock.toString()
+                .contains(":") || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("FIGURE") || textBlock.toString()
+                .startsWith("TABLE")) && !textBlock.toString().endsWith(":") && matcher2.find()) {
+            textBlock.setClassification(PageBlockType.getHeadlineType(1));
+            document.setHeadlines(true);
+
+        } else if (matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && matcher2.find() && !matcher3.matches()) {
+            textBlock.setClassification(PageBlockType.getHeadlineType(2));
+            document.setHeadlines(true);
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
            textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
@@ -6,6 +6,7 @@ import java.util.regex.Pattern;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -27,8 +28,8 @@ public class RedactManagerClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true);
+        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.REDACT_MANAGER);
+        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.REDACT_MANAGER);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
@@ -6,6 +6,7 @@ import java.util.regex.Pattern;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -27,8 +28,8 @@ public class TaasClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false);
-        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true);
+        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.TAAS);
+        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.TAAS);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
@@ -149,7 +149,7 @@ public class RectangleTransformations {
        @Override
        public BiConsumer<BBox, Rectangle2D> accumulator() {

-            return (bb, rect) -> bb.addRectangle(rect.getMinX(), rect.getMinY(), rect.getMaxX(), rect.getMaxY());
+            return BBox::addRectangle;
        }


@@ -166,7 +166,7 @@ public class RectangleTransformations {
        @Override
        public Function<BBox, Rectangle2D> finisher() {

-            return bb -> new Rectangle2D.Double(bb.lowerLeftX, bb.lowerLeftY, bb.upperRightX - bb.lowerLeftX, bb.upperRightY - bb.lowerLeftY);
+            return BBox::toRectangle2D;
        }


@@ -187,7 +187,21 @@ public class RectangleTransformations {
            Double upperRightY;


-            public void addRectangle(double lowerLeftX, double lowerLeftY, double upperRightX, double upperRightY) {
+            public Rectangle2D toRectangle2D() {
+
+                if (lowerLeftX == null || lowerLeftY == null || upperRightX == null || upperRightY == null) {
+                    return new Rectangle2D.Double(0, 0, 0, 0);
+                }
+                return new Rectangle2D.Double(lowerLeftX, lowerLeftY, upperRightX - lowerLeftX, upperRightY - lowerLeftY);
+            }
+
+
+            public void addRectangle(Rectangle2D rectangle2D) {
+
+                double lowerLeftX = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX());
+                double lowerLeftY = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY());
+                double upperRightX = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX());
+                double upperRightY = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY());

                if (this.lowerLeftX == null) {
                    this.lowerLeftX = lowerLeftX;
@@ -1,20 +1,27 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;

-import java.util.Comparator;
 import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.util.QuickSort;

 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

 public class TextPositionOperations {

+    private static final TextPositionSequenceComparator comparator = new TextPositionSequenceComparator();
+
+
    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<TextPageBlock> textBlocks) {

-        return textBlocks.stream()//
-                .flatMap(tb -> tb.getSequences().stream())//
-                .sorted(Comparator.comparingDouble(TextPositionSequence::getMaxYDirAdj)//
-                        .thenComparing(TextPositionSequence::getMaxXDirAdj))//
-                .toList();
+        var sequence = textBlocks.stream().flatMap(tb -> tb.getSequences().stream()).collect(Collectors.toList());
+
+        // because the TextPositionSequenceComparator is not transitive, but
+        // JDK7+ enforces transitivity on comparators, we need to use
+        // a custom quicksort implementation (which is slower, unfortunately).
+        QuickSort.sort(sequence, comparator);
+        return sequence;
    }

 }
@@ -0,0 +1,91 @@
+import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
+
+plugins {
+    id("com.knecon.fforesight.java-conventions")
+    id("org.springframework.boot") version "3.1.2"
+    id("io.spring.dependency-management") version "1.1.0"
+    id("org.sonarqube") version "4.2.1.3168"
+    id("io.freefair.lombok") version "8.1.0"
+    id("org.graalvm.buildtools.native") version "0.9.23"
+}
+
+dependencies {
+    implementation(project(":layoutparser-service-processor"))
+
+    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+
+    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
+    implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
+
+
+    // for integration testing only
+    testImplementation(project(":layoutparser-service-internal-api"))
+
+    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
+    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
+    testImplementation("com.iqser.red.commons:jackson-commons:1.0.0")
+    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
+    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
+    testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
+    testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
+    testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
+}
+
+description = "layoutparser-service-server"
+
+java {
+    withJavadocJar()
+}
+// AOT seems to be the name of the generated classes for native images
+// They are added as a SourceSet, and therefore checkstyle and pmd try to run on it
+tasks.named<Checkstyle>("checkstyleAot") {
+    enabled = false
+}
+tasks.named<Checkstyle>("checkstyleAotTest") {
+    enabled = false
+}
+tasks.named<Pmd>("pmdAot") {
+    enabled = false
+}
+tasks.named<Pmd>("pmdAotTest") {
+    enabled = false
+}
+
+
+graalvmNative {
+    nativeBuild {
+        resources.autodetect()
+        buildArgs.add("--verbose")
+        buildArgs.add("--add-opens=java.base/java.nio=ALL-UNNAMED")
+        buildArgs.add("--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED")
+        buildArgs.add("--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED")
+        buildArgs.add("--trace-class-initialization=ch.qos.logback.classic.Logger")
+        buildArgs.add("--trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase\$Worker")
+        buildArgs.add("--initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback")
+        buildArgs.add("--initialize-at-run-time=io.netty")
+        buildArgs.add("-H:ReflectionConfigurationFiles=../../../src/main/resources/reflection-config.json")
+    }
+}
+
+
+tasks.named<BootBuildImage>("bootBuildImage") {
+    imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
+    if (project.hasProperty("buildbootDockerHostNetwork")) {
+        network.set("host")
+    }
+    docker {
+        if (project.hasProperty("buildbootDockerHostNetwork")) {
+            bindHostToBuilder.set(true)
+        }
+        verboseLogging.set(true)
+
+        publishRegistry {
+            username.set(providers.gradleProperty("mavenUser").getOrNull())
+            password.set(providers.gradleProperty("mavenPassword").getOrNull())
+            email.set(providers.gradleProperty("mavenEmail").getOrNull())
+            url.set("https://nexus.knecon.com:5001/")
+        }
+    }
+}
@@ -1,84 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.knecon.fforesight</groupId>
-        <artifactId>layoutparser-service</artifactId>
-        <version>0.1-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>layoutparser-service-server</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.knecon.fforesight</groupId>
-            <artifactId>layoutparser-service-processor</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-amqp</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-starter-openfeign</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-actuator</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-test</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-slf4j-impl</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-
-            <plugin>
-                <!-- generate git.properties for exposure in /info -->
-                <groupId>pl.project13.maven</groupId>
-                <artifactId>git-commit-id-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>revision</goal>
-                        </goals>
-                        <configuration>
-                            <generateGitPropertiesFile>true</generateGitPropertiesFile>
-                            <gitDescribe>
-                                <tags>true</tags>
-                            </gitDescribe>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <!-- repackages the generated jar into a runnable fat-jar and makes it
-                    executable -->
-                <groupId>org.springframework.boot</groupId>
-                <artifactId>spring-boot-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>repackage</goal>
-                        </goals>
-                        <configuration>
-                            <executable>true</executable>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
@@ -0,0 +1,112 @@
+[
+  {
+    "name": "ch.qos.logback.classic.AsyncAppender",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.classic.encoder.PatternLayoutEncoder",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.classic.pattern.DateConverter",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.classic.pattern.LevelConverter",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.classic.pattern.LineSeparatorConverter",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.classic.pattern.LoggerConverter",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.classic.pattern.MessageConverter",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.classic.pattern.ThreadConverter",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.core.ConsoleAppender",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "ch.qos.logback.core.FileAppender",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true,
+    "allDeclaredClasses": true,
+    "allPublicClasses": true
+  }
+]
@@ -6,8 +6,9 @@ import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;

 class ApplicationTests extends BaseTest {

-	@Test
-	void contextLoads() {
-	}
+    @Test
+    void contextLoads() {
+
+    }

 }
@@ -45,8 +45,9 @@ public class BdrJsonBuildTest extends BaseTest {
    protected Document buildGraph(File filename) {

        try (InputStream inputStream = new FileInputStream(filename)) {
-            PDDocument pdDocument = Loader.loadPDF(inputStream);
-            return layoutParsingPipeline.parseLayoutWithTimer(LayoutParsingType.REDACT_MANAGER, pdDocument, new ImageServiceResponse(), new TableServiceResponse());
+            try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
+                return layoutParsingPipeline.parseLayoutWithTimer(LayoutParsingType.TAAS, pdDocument, new ImageServiceResponse(), new TableServiceResponse());
+            }
        }
    }

@@ -94,12 +95,13 @@ public class BdrJsonBuildTest extends BaseTest {

    private static void visualizeSemanticNodes(File file, File resultingFileName, Document document, TextBlock textBlock) throws IOException {

-        try (var fileStream = new FileInputStream(file); var outputStream = new FileOutputStream(resultingFileName)) {
-            PDDocument pdDocument = Loader.loadPDF(fileStream);
+        try (var fileStream = new FileInputStream(file);//
+             PDDocument pdDocument = Loader.loadPDF(fileStream);//
+             var outputStream = new FileOutputStream(resultingFileName)//
+        ) {
            PdfDraw.drawDocumentGraph(pdDocument, document);
            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
            pdDocument.save(outputStream);
-            pdDocument.close();
        }
    }

@@ -0,0 +1,157 @@
+package com.knecon.fforesight.service.layoutparser.server;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.Loader;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.tenantcommons.TenantsClient;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.SneakyThrows;
+import lombok.ToString;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(HeadlinesGoldStandardIntegrationTest.RedactionIntegrationTestConfiguration.class)
+public class HeadlinesGoldStandardIntegrationTest {
+
+    @MockBean
+    private TenantsClient tenantsClient;
+
+    @MockBean
+    private RabbitTemplate rabbitTemplate;
+
+    @Autowired
+    private ObjectMapper objectMapper;
+
+    @Autowired
+    private StorageService storageService;
+
+    @Autowired
+    private LayoutParsingPipeline layoutParsingPipeline;
+
+
+    @Test
+    public void testHeadlineDetection() {
+
+        List<Metrics> metrics = new ArrayList<>();
+        metrics.add(getMetrics("files/headlineTest/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf",
+                "files/headlineTest/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1)_REDACTION_LOG.json"));
+        metrics.add(getMetrics("files/headlineTest/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf",
+                "files/headlineTest/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23_REDACTION_LOG.json"));
+        metrics.add(getMetrics("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", "files/headlineTest/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
+
+        double precision = metrics.stream().mapToDouble(Metrics::getPrecision).average().orElse(1.0);
+        double recall = metrics.stream().mapToDouble(Metrics::getRecall).average().orElse(1.0);
+
+        System.out.println("Precision is: " + precision + " recall is: " + recall);
+
+        Assertions.assertThat(precision).isGreaterThanOrEqualTo(0.44f);
+        Assertions.assertThat(recall).isGreaterThanOrEqualTo(0.69f);
+    }
+
+
+    @SneakyThrows
+    private Metrics getMetrics(String filePath, String redactionLogUrl) {
+
+        ClassPathResource redactionLogResource = new ClassPathResource(redactionLogUrl);
+        ClassPathResource pdfFileResource = new ClassPathResource(filePath);
+
+        Set<Headline> goldStandardHeadlines = new HashSet<>();
+        var goldStandardLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class);
+        goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
+        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));
+
+        Document documentGraph = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                Loader.loadPDF(pdfFileResource.getInputStream()),
+                new ImageServiceResponse(),
+                new TableServiceResponse());
+
+        var foundHeadlines = documentGraph.streamAllSubNodes()
+                .map(SemanticNode::getHeadline)
+                .distinct()
+                .map(headlineNode -> new Headline(headlineNode.getPages().stream().findFirst().get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing()))
+                .toList();
+
+        Set<Headline> correct = new HashSet<>();
+        Set<Headline> missing;
+        Set<Headline> falsePositive = new HashSet<>();
+        for (Headline headline : foundHeadlines) {
+            if (goldStandardHeadlines.contains(headline)) {
+                correct.add(headline);
+            } else {
+                falsePositive.add(headline);
+            }
+        }
+
+        missing = goldStandardHeadlines.stream().filter(h -> !correct.contains(h)).collect(Collectors.toSet());
+
+        float precision = (float) correct.size() / (float) foundHeadlines.size();
+        float recall = (float) correct.size() / ((float) correct.size() + (float) missing.size());
+
+        return new Metrics(precision, recall);
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class/*, StorageAutoConfiguration.class*/})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    public static class RedactionIntegrationTestConfiguration {
+
+    }
+
+    @Data
+    @EqualsAndHashCode
+    @AllArgsConstructor
+    @ToString
+    private class Metrics {
+
+        private float precision;
+        private float recall;
+
+    }
+
+    @Data
+    @EqualsAndHashCode
+    @AllArgsConstructor
+    @ToString
+    private class Headline {
+
+        private int page;
+        private String headline;
+
+    }
+
+}
@@ -22,7 +22,8 @@ import lombok.SneakyThrows;
 public class BuildDocumentGraphTest extends BaseTest {

    @Autowired
-    private LayoutParsingPipeline layoutParsingPipeline;
+    protected LayoutParsingPipeline layoutParsingPipeline;
+

    @Test
    @Disabled
@@ -30,28 +31,26 @@ public class BuildDocumentGraphTest extends BaseTest {

        Document documentGraph = buildGraph("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
        assertEquals(221, documentGraph.getPages().size());
-        assertEquals(220 , documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
-        assertEquals(0 , documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
+        assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
+        assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
    }


    @SneakyThrows
    protected Document buildGraph(String filename) {

-        if (!filename.endsWith(".pdf")) {
-            filename = filename + ".pdf";
-        }
-
-        if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
+        if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
            prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json");
        } else {
            prepareStorage(filename);
        }
        ClassPathResource fileResource = new ClassPathResource(filename);

-        try (InputStream inputStream = fileResource.getInputStream()) {
-            PDDocument pdDocument = Loader.loadPDF(inputStream);
-            return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse());
+        try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream)) {
+            return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                    pdDocument,
+                    layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
+                    new TableServiceResponse());
        }
    }

@@ -1,14 +1,22 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;

+import org.apache.pdfbox.Loader;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;

@@ -17,22 +25,50 @@ import lombok.SneakyThrows;
 public class DocumentGraphJsonWritingTest extends BuildDocumentGraphTest {

    @Test
+    @SneakyThrows
+    @Disabled
+    public void writeJsonForAllFilesTest() {
+
+        Path path = Path.of("/tmp/test_files");
+
+        Files.walk(path)
+                .map(Path::toFile)
+                .filter(File::isFile)
+                .filter(file -> file.getName().endsWith(".pdf"))
+                .peek(System.out::println)
+                .map(File::toPath)
+                .forEach(this::writeJsons);
+    }
+
+
+    @Test
+    @SneakyThrows
    @Disabled
    public void writeJsonForFileTest() {

-        writeJsons("files/216");
+        var resource = new ClassPathResource("files/1 Abamectin_prr.pdf");
+        writeJsons(resource.getFile().toPath());
    }

-    @SneakyThrows
-    private void writeJsons(String filename) {

-        Document documentGraph = buildGraph(filename);
+    @SneakyThrows
+    private void writeJsons(Path filename) {
+
+        Document documentGraph = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                Loader.loadPDF(new FileInputStream(filename.toFile())),
+                new ImageServiceResponse(),
+                new TableServiceResponse());
+
        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
-        mapper.writeValue(new FileOutputStream(File.createTempFile(filename + "_structure", ".json")), documentData.getDocumentTreeData());
-        mapper.writeValue(new FileOutputStream(File.createTempFile(filename + "_text", ".json")), documentData.getAtomicTextBlocks());
-        mapper.writeValue(new FileOutputStream(File.createTempFile(filename + "_positions", ".json")), documentData.getAtomicPositionBlocks());
-        mapper.writeValue(new FileOutputStream(File.createTempFile(filename + "_pages", ".json")), documentData.getPages());
+
+        var stem = Path.of("/tmp/DocumentGraphJsonWritingTest");
+        stem.toFile().mkdirs();
+        var tmpFilePath = stem.resolve(filename.getFileName());
+        mapper.writeValue(new FileOutputStream(new File(tmpFilePath + "_structure" + ".json")), documentData.getDocumentStructure());
+        mapper.writeValue(new FileOutputStream(new File(tmpFilePath + "_text" + ".json")), documentData.getDocumentTextData());
+        mapper.writeValue(new FileOutputStream(new File(tmpFilePath + "_positions" + ".json")), documentData.getDocumentPositions());
+        mapper.writeValue(new FileOutputStream(new File(tmpFilePath + "_pages" + ".json")), documentData.getDocumentPages());
    }

 }
@@ -1,14 +1,22 @@
 package com.knecon.fforesight.service.layoutparser.server.graph;

-import org.junit.jupiter.api.Disabled;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentGraphMapper;
-import com.knecon.fforesight.service.layoutparser.processor.mapper.taas.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.PropertiesMapper;
+import com.knecon.fforesight.tenantcommons.TenantContext;

 import lombok.SneakyThrows;

@@ -16,22 +24,64 @@ public class DocumentGraphMappingTest extends BuildDocumentGraphTest {

    @Test
    @SneakyThrows
-    @Disabled
    public void testGraphMapping() {

-        Document document = buildGraph("files/crafted document");
-        LayoutParsingRequest layoutParsingRequest = buildStandardLayoutParsingRequest();
+        String filename = "files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf";

+        Document document = buildGraph(filename);
        DocumentData documentData = DocumentDataMapper.toDocumentData(document);
-        var researchDocumentData = TaasDocumentDataMapper.fromDocument(document);

-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, documentData);
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getDocumentPages());
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getDocumentTextData());
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_POSITIONS" + ".json", documentData.getDocumentPositions());
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", documentData.getDocumentStructure());

-        DocumentData documentData2 = layoutParsingStorageService.readDocumentData(layoutParsingRequest);
-        Document newDocumentGraph = DocumentGraphMapper.toDocumentGraph(documentData2);
+        DocumentPage[] pageData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", DocumentPage[].class);
+        DocumentTextData[] atomicTextBlockData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", DocumentTextData[].class);
+        DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
+                filename + "_POSITIONS" + ".json",
+                DocumentPositionData[].class);
+        DocumentStructure documentTreeData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", DocumentStructure.class);

-        assert document.toString().equals(newDocumentGraph.toString());
-        assert document.getDocumentTree().toString().equals(newDocumentGraph.getDocumentTree().toString());
+        DocumentData documentData2 = DocumentData.builder()
+                .documentPages(pageData)
+                .documentStructure(documentTreeData)
+                .documentTextData(atomicTextBlockData)
+                .documentPositions(atomicPositionBlockData)
+                .build();
+        Document newDocument = DocumentGraphMapper.toDocumentGraph(documentData2);
+
+        assertTrue(allTablesHavePositiveNumberOfRowsAndColumns(document));
+        assertTrue(allTablesHavePositiveNumberOfRowsAndColumns(documentData));
+        assertTrue(allTablesHavePositiveNumberOfRowsAndColumns(documentData2));
+        assertTrue(allTablesHavePositiveNumberOfRowsAndColumns(newDocument));
+
+        assertEquals(document.toString(), newDocument.toString());
+        assertEquals(document.getDocumentTree().toString(), newDocument.getDocumentTree().toString());
+    }
+
+
+    private static boolean allTablesHavePositiveNumberOfRowsAndColumns(Document document) {
+
+        return document.streamAllSubNodes()
+                .filter(semanticNode -> semanticNode instanceof Table)
+                .map(semanticNode -> (Table) semanticNode)
+                .allMatch(tableNode -> tableNode.getNumberOfCols() > 0 && tableNode.getNumberOfRows() > 0);
+    }
+
+
+    private static boolean allTablesHavePositiveNumberOfRowsAndColumns(DocumentData documentData) {
+
+        return documentData.getDocumentStructure()
+                .streamAllEntries()
+                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
+                .map(DocumentStructure.EntryData::getProperties)
+                .map(properties -> {
+                    var builder = Table.builder();
+                    PropertiesMapper.parseTableProperties(properties, builder);
+                    return builder.build();
+                })
+                .allMatch(tableNode -> tableNode.getNumberOfCols() > 0 && tableNode.getNumberOfRows() > 0);
    }

 }
@@ -63,12 +63,12 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
        File tmpFile = File.createTempFile(filename, "SEMANTIC_NODES_BBOX.pdf");
        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");

-        try (var fileStream = fileResource.getInputStream()) {
-            PDDocument pdDocument = Loader.loadPDF(fileStream);
-            PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
-            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
-            pdDocument.save(tmpFile);
-            pdDocument.close();
+        try (var fileStream = fileResource.getInputStream();//
+             PDDocument pdDocument = Loader.loadPDF(fileStream)//
+        ) {
+                PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
+                PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
+                pdDocument.save(tmpFile);
        }
    }

@@ -0,0 +1,60 @@
+package com.knecon.fforesight.service.layoutparser.server.model;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.pdfbox.util.Matrix;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.storage.commons.properties.StorageProperties;
+import com.iqser.red.storage.commons.service.ObjectSerializer;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.SneakyThrows;
+
+public class TextPositionSequenceTest {
+
+    private static final String TEXT_POSITION_SEQUENCE_AS_JSON = "{\n" //
+            + "  \"page\": 1,\n" //
+            + "  \"textPositions\": [],\n" //
+            + "  \"dir\": 180.0,\n" //
+            + "  \"rotation\": 0,\n" //
+            + "  \"pageHeight\": 800,\n" //
+            + "  \"pageWidth\": 600\n"  //
+            + "}";
+
+    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper(), new StorageProperties());
+
+
+    @Test
+    @SneakyThrows
+    public void testDeserializationWithJackson() {
+
+        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithJackson(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+                TextPositionSequence.class);
+
+        assertPropertiesAfterJsonDeserialization(textPositionSequence);
+    }
+
+
+    private void assertPropertiesAfterJsonDeserialization(TextPositionSequence textPositionSequence) {
+
+        assertThat(textPositionSequence.getPage()).isEqualTo(1);
+        assertThat(textPositionSequence.getTextPositions()).hasSize(0);
+        assertThat(textPositionSequence.getDir()).isEqualTo(TextDirection.HALF_CIRCLE);
+        assertThat(textPositionSequence.getRotation()).isEqualTo(0);
+        assertThat(textPositionSequence.getPageHeight()).isEqualTo(800f);
+        assertThat(textPositionSequence.getPageWidth()).isEqualTo(600f);
+    }
+
+
+    private Matrix createIdentityMatrix() {
+
+        return new Matrix();
+    }
+
+}
@@ -0,0 +1,531 @@
+package com.knecon.fforesight.service.layoutparser.server.segmentation;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.awt.geom.Rectangle2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
+import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
+
+import lombok.SneakyThrows;
+
+public class PdfSegmentationServiceTest extends BaseTest {
+
+    @Autowired
+    private PdfParsingService pdfParsingService;
+
+    @Autowired
+    private ObjectMapper objectMapper;
+
+    @Autowired
+    private RedactManagerClassificationService redactManagerClassificationService;
+
+    @Autowired
+    private CvTableParsingAdapter cvTableParsingAdapter;
+
+    @Autowired
+    private ImageServiceResponseAdapter imageServiceResponseAdapter;
+
+    @Autowired
+    private SectionsBuilderService sectionsBuilderService;
+
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
+
+        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(LayoutParsingType.REDACT_MANAGER,
+                originDocument,
+                cvTableParsingAdapter.buildCvParsedTablesPerPage(new TableServiceResponse()),
+                imageServiceResponseAdapter.buildClassifiedImagesPerPage(new ImageServiceResponse()));
+
+        redactManagerClassificationService.classifyDocument(classificationDocument);
+
+        sectionsBuilderService.buildSections(classificationDocument);
+
+        return classificationDocument;
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testMapping() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/image_response.json");
+        ImageServiceResponse imageServiceResponse = objectMapper.readValue(responseJson.getInputStream(), ImageServiceResponse.class);
+
+        Map<Integer, List<ClassifiedImage>> images = new HashMap<>();
+        imageServiceResponse.getData()
+                .forEach(imageMetadata -> images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
+                        .add(new ClassifiedImage(new Rectangle2D.Double(imageMetadata.getPosition().getX1(),
+                                imageMetadata.getPosition().getY1(),
+                                imageMetadata.getGeometry().getWidth(),
+                                imageMetadata.getGeometry().getHeight()),
+                                ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)),
+                                imageMetadata.isAlpha(),
+                                imageMetadata.getPosition().getPageNumber())));
+
+        System.out.println("object");
+    }
+
+
+    @Test
+    public void testPDFSegmentationWithComplexTable() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        assertThat(table.getColCount()).isEqualTo(6);
+        assertThat(table.getRowCount()).isEqualTo(13);
+        assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
+    }
+
+
+    @Test
+    public void testTableExtraction() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        assertThat(firstTable.getColCount()).isEqualTo(8);
+        assertThat(firstTable.getRowCount()).isEqualTo(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        assertThat(secondTable.getColCount()).isEqualTo(8);
+        assertThat(secondTable.getRowCount()).isEqualTo(2);
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
+    }
+
+
+    @Test
+    public void testMultiPageMetadataPropagation() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        assertThat(firstTable.getColCount()).isEqualTo(9);
+        assertThat(firstTable.getRowCount()).isEqualTo(5);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        assertThat(secondTable.getColCount()).isEqualTo(9);
+        assertThat(secondTable.getRowCount()).isEqualTo(6);
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
+    }
+
+
+    @Test
+    public void testHeaderCellsForRotatedTable() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        assertThat(firstTable.getColCount()).isEqualTo(8);
+        assertThat(firstTable.getRowCount()).isEqualTo(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        assertThat(secondTable.getColCount()).isEqualTo(8);
+        assertThat(secondTable.getRowCount()).isEqualTo(6);
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
+    }
+
+
+    @Test
+    public void testDoc56Page170() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 1, 1, 0, 0);
+        validateTable(document, 1, 2, 2, 0, 0);
+        validateTable(document, 2, 7, 20, 0, 140);
+        validateTable(document, 3, 8, 31, 0, 170);
+
+    }
+
+
+    @Test
+    public void testVV931175Page1() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 8, 8, 0, 2);
+
+        List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
+                        "Author, date",
+                        "Study title",
+                        "Analytical method Author, date, No.",
+                        "Technique, LOQ of the method, validated working range",
+                        "Method meets analytical validation criteria",
+                        "Remarks (in case validation criteria are not met)",
+                        "Acceptability of the method"),
+                Arrays.asList("",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
+                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
+                        "Evans P.G. 2001 TMJ4569B, VV-323245",
+                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
+                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
+                        "Y",
+                        "N/A",
+                        "Y"));
+
+        validateTable(document, 0, values);
+
+    }
+
+
+    @Test
+    public void testDoc27Page6() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 3, 2, 0, 0);
+        validateTable(document, 1, 3, 2, 0, 0);
+        validateTable(document, 2, 3, 3, 0, 0);
+        validateTable(document, 3, 3, 3, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDocA20622APartB9Page185() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 5, 5, 0, 23);
+        validateTable(document, 1, 11, 9, 0, 36);
+
+    }
+
+
+    @Test
+    public void testDocA20622APartB7Page123() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izZRMS (CZ) fRR Part B7_Page123.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 6);
+
+        validateTable(document, 0, 2, 1, 0, 0);
+        validateTable(document, 1, 2, 1, 0, 0);
+        validateTable(document, 2, 2, 5, 0, 0);
+        validateTable(document, 3, 2, 5, 0, 0);
+        validateTable(document, 4, 2, 4, 0, 0);
+        validateTable(document, 5, 2, 1, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc77Page111() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04_Page11.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 3);
+
+        validateTable(document, 0, 7, 9, 0, 0);
+        validateTable(document, 1, 2, 1, 0, 0);
+        validateTable(document, 2, 2, 10, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc95Page532() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10_Page532.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 1);
+        validateTable(document, 0, 9, 9, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc52Page175() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 9, 5, 6, 0);
+
+    }
+
+
+    @Test
+    public void testDoc52Page174() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 1);
+        validateTable(document, 0, 9, 6, 7, 0);
+
+    }
+
+
+    @Test
+    public void testDoc19Page35() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 1);
+        validateTable(document, 0, 10, 6, 0, 1);
+
+    }
+
+
+    @Test
+    public void testDoc19Page161() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 2);
+        validateTable(document, 0, 2, 2, 0, 0);
+        validateTable(document, 1, 1, 1, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc47Page30() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(
+                "files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 7, 8, 1, 0);
+        validateTable(document, 1, 7, 8, 1, 0);
+
+    }
+
+
+    @Test
+    public void testDoc49Page61() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(
+                "files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 4, 17, 0, 0);
+        validateTable(document, 1, 7, 12, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc81Page54() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 5, 14, 4, 0);
+        validateTable(document, 1, 7, 12, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc88Page134() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 5, 17, 3, 0);
+        validateTable(document, 1, 5, 16, 2, 0);
+
+    }
+
+
+    @Test
+    public void testDocThiabendazolePage18() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 4, 4, 0, 0);
+        validateTable(document, 1, 1, 1, 0, 0);
+        validateTable(document, 2, 2, 3, 0, 0);
+        validateTable(document, 3, 1, 1, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc15Page18() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 11, 8, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc28Page23() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(
+                "files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 6, 8, 0, 2);
+        validateTable(document, 1, 6, 8, 0, 1);
+
+    }
+
+
+    @Test
+    public void testDoc24Page17() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 9, 5, 2, 0);
+
+    }
+
+
+    @Test
+    public void testDoc30Page5() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getInputStream()));
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 3, 5, 0, 0);
+
+    }
+
+
+    private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {
+
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        List<List<Cell>> rows = table.getRows();
+        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
+
+        assertThat(emptyCellsFoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);
+
+        assertThat(table.getColCount()).isEqualTo(colCount);
+        assertThat(table.getRowCount()).isEqualTo(rowCount);
+
+    }
+
+
+    private void validateTable(ClassificationDocument document, int tableIndex, List<List<String>> values) {
+
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        List<List<Cell>> rows = table.getRows();
+
+        List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();
+        List<String> valuesFlattened = values.stream().flatMap(List::stream).toList();
+
+        for (int i = 0; i < valuesFlattened.size(); i++) {
+            Cell cell = rowsFlattened.get(i);
+            String value = valuesFlattened.get(i);
+            assertThat(cell.toString()).isEqualTo(value);
+        }
+
+    }
+
+
+    private void validateTableSize(ClassificationDocument document, int tableSize) {
+
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().size()).isEqualTo(tableSize);
+
+    }
+
+}
@@ -26,7 +26,7 @@ class GapAcrossLinesDetectionServiceTest {
    @SneakyThrows
    public void testGapBasedColumnDetection() {

-        String filename = "files/211.pdf";
+        String filename = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_COLUMNS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
@@ -52,7 +52,7 @@ class GapAcrossLinesDetectionServiceTest {
    @SneakyThrows
    public void testColumnDetection() {

-        String filename = "files/211.pdf";
+        String filename = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_COLUMNS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
@@ -7,15 +7,14 @@ import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;

-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.InvisibleTableDetectionService;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageInformationService;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.services.TextPositionSequenceSorter;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

 import lombok.SneakyThrows;
@@ -23,11 +22,11 @@ import lombok.SneakyThrows;
 class InvisibleTableDetectionServiceTest {

    @Test
-    @Disabled
+//    @Disabled
    @SneakyThrows
    public void detectInvisibleTableTest() {

-        String fileName = "files/211.pdf";
+        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TABLE.pdf").toString();
        List<PageInformation> pageContents = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName).stream().map(PageInformationService::build).collect(Collectors.toList());

@@ -18,7 +18,7 @@ class MainBodyTextFrameExtractionServiceTest {
    @SneakyThrows
    public void testMainBodyDetection() {

-        String fileName = "files/211.pdf";
+        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
        String tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_MAIN_BODY.pdf").toString();
        List<PageContents> sortedTextPositionSequence = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName);

@@ -20,7 +20,7 @@ class PageInformationServiceTest {
    @SneakyThrows
    public void testGapDetection() {

-        String filename = "files/211.pdf";
+        String filename = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_GAPS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
@@ -43,7 +43,7 @@ class PageInformationServiceTest {
    @SneakyThrows
    public void testLineDetection() {

-        String filename = "files/211.pdf";
+        String filename = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_GAPS.pdf";
        System.out.println("start TextPosition extraction");
        long start = System.currentTimeMillis();
@@ -21,7 +21,7 @@ class TextPositionSequenceSorterTest {
    @SneakyThrows
    public void testTextPositionSequenceExtraction() {

-        String fileName = "files/211.pdf";
+        String fileName = "files/invisible_tables/test-two-pages_ocred.pdf";
        var tmpFileName = Path.of("/tmp/").resolve(Path.of(fileName).getFileName() + "_TEXT_POSITION_SEQUENCES.pdf").toString();

        List<PageContents> textPositionPerPage = TextPositionSequenceSorter.getSortedTextPositionsWithPages(fileName);
@@ -12,17 +12,15 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;

 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
 import com.knecon.fforesight.service.layoutparser.server.Application;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
@@ -151,13 +149,6 @@ public class BaseTest {
    @ComponentScan("com.knecon.fforesight.service.layoutparser")
    public static class TestConfiguration {

-        @Bean
-        @Primary
-        public StorageService inMemoryStorage() {
-
-            return new FileSystemBackedStorageService();
-        }
-
    }

 }
@@ -24,14 +24,13 @@ import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document
 import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Getter;
-import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.experimental.UtilityClass;
@@ -41,20 +40,17 @@ public class PdfDraw {

    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) throws IOException {

-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
-            PDDocument pdDocument = Loader.loadPDF(inputStream);
-
+        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
+             PDDocument pdDocument = Loader.loadPDF(inputStream);//
+             var out = new FileOutputStream(tmpFileName)//
+        ) {
            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
                        pageNumber,
                        rectanglesPerPage.get(pageNumber - 1),
                        PdfVisualisationUtility.Options.builder().stroke(true).build());
            }
-            try (var out = new FileOutputStream(tmpFileName)) {
-                pdDocument.save(out);
-                pdDocument.close();
-            }
-
+            pdDocument.save(out);
        }

    }
@@ -62,8 +58,10 @@ public class PdfDraw {

    public static void drawRectanglesPerPageNumberedByLine(String filename, List<List<List<Rectangle2D>>> rectanglesPerPage, String tmpFileName) throws IOException {

-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
-            PDDocument pdDocument = Loader.loadPDF(inputStream);
+        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
+             PDDocument pdDocument = Loader.loadPDF(inputStream);//
+             var out = new FileOutputStream(tmpFileName)//
+        ) {

            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
                var rectanglesOnPage = rectanglesPerPage.get(pageNumber - 1);
@@ -80,10 +78,8 @@ public class PdfDraw {
                }

            }
-            try (var out = new FileOutputStream(tmpFileName)) {
-                pdDocument.save(out);
-                pdDocument.close();
-            }
+
+            pdDocument.save(out);

        }

@@ -92,15 +88,18 @@ public class PdfDraw {

    private static int countNumberOfDigits(int num) {

-        if (num == 0) {
+        int final_num = num;
+        if (final_num == 0) {
            return 1;
        }
        int count = 0;
-        for (; num != 0; num /= 10, ++count) {
+        for (; final_num != 0; final_num /= 10) {
+            count++;
        }
        return count;
    }

+
    public static void drawDocumentGraph(PDDocument document, Document documentGraph) {

        documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry));
@@ -183,10 +182,12 @@ public class PdfDraw {


    @SneakyThrows
-    public static void drawRectanglesAndLinesPerPage(String filename, List<List<Rectangle2D>> list,  List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) {
+    public static void drawRectanglesAndLinesPerPage(String filename, List<List<Rectangle2D>> list, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) {

-        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
-            PDDocument pdDocument = Loader.loadPDF(inputStream);
+        try (InputStream inputStream = new ClassPathResource(filename).getInputStream();//
+             PDDocument pdDocument = Loader.loadPDF(inputStream);//
+             var out = new FileOutputStream(tmpFileName)//
+        ) {

            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
 //                PdfVisualisationUtility.drawLine2DList(pdDocument,
@@ -197,15 +198,9 @@ public class PdfDraw {
                        pageNumber,
                        rectanglesPerPage.get(pageNumber - 1),
                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
-                        pageNumber,
-                        list.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-            }
-            try (var out = new FileOutputStream(tmpFileName)) {
-                pdDocument.save(out);
-                pdDocument.close();
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().stroke(true).build());
            }
+            pdDocument.save(out);

        }
    }
@@ -213,19 +208,17 @@ public class PdfDraw {

    @Builder
    @AllArgsConstructor
-    @NoArgsConstructor
    @Getter
    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
    public static class Options {

-        @Builder.Default
-        boolean stroke = false;
+        boolean stroke;
        @Builder.Default
        Color strokeColor = Color.BLACK;
        @Builder.Default
        float strokeWidth = 1f;
-        @Builder.Default
-        boolean fill = false;
+
+        boolean fill;
        @Builder.Default
        Color fillColor = Color.BLACK;

@@ -250,13 +243,19 @@ public class PdfDraw {
    private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, Options options) {

        Map<Page, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
-        rectanglesPerPage.forEach((page, rectangle2D) -> {
+        for (Page page : rectanglesPerPage.keySet()) {
+            Rectangle2D rectangle2D = rectanglesPerPage.get(page);
            if (entry.getType() == NodeType.SECTION) {
                rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
            }
            drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options);
-            drawText(buildString(entry), document, new Point2D.Double(rectangle2D.getMinX(), rectangle2D.getMaxY() + 2), page.getNumber(), options, entry.getType() == NodeType.TABLE_CELL);
-        });
+            drawText(buildString(entry),
+                    document,
+                    new Point2D.Double(rectangle2D.getMinX(), rectangle2D.getMaxY() + 2),
+                    page.getNumber(),
+                    options,
+                    entry.getType() == NodeType.TABLE_CELL);
+        }
    }


@@ -31,7 +31,3 @@ management:
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
  endpoints.web.exposure.include: prometheus, health
-
-
-storage:
-  backend: 's3'
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Kilian Schuettler	f57c27a7e5	try native build	2023-07-27 13:50:32 +02:00
Kilian Schüttler	2ea58f5e9f	Merge branch 'fixGradleDeploy' into 'main' Fix gradle deploy See merge request fforesight/layout-parser!3	2023-07-27 13:42:10 +02:00
Kilian Schuettler	510ec7ce45	remove native build	2023-07-27 13:29:14 +02:00
Kilian Schuettler	c186927e3d	add reflection config	2023-07-27 13:21:01 +02:00
Kilian Schuettler	1a494b0dea	add buildArgs to include logback at runtime	2023-07-27 13:00:26 +02:00
Kilian Schuettler	19552ddf69	added gradle.properties.kts	2023-07-27 12:32:10 +02:00
Kilian Schuettler	41267a0f98	ported to gradle	2023-07-27 12:27:30 +02:00
Kilian Schuettler	270129cd73	outputs almost equal current redaction-service in regards to RedactManager * 3/200 files have minimal whitespace/sorting errors, most likely rounding errors	2023-07-25 18:12:57 +02:00
Kilian Schuettler	a41c13fdd6	add LayoutParsingType to BodyTextFrameService	2023-07-25 17:20:52 +02:00
Kilian Schuettler	65ab5eca22	update to redaction-service state	2023-07-25 16:10:57 +02:00
Kilian Schuettler	143ebee25e	move and fix layout tests from redaction-service	2023-07-24 19:43:25 +02:00
Kilian Schuettler	47fd8e05d1	rename Data classes	2023-07-24 18:36:27 +02:00