try native build

2023-07-27 13:50:32 +02:00
140 changed files with 988 additions and 2050 deletions
@@ -38,7 +38,3 @@ build/

 ### VS Code ###
 .vscode/
-gradlew.bat
-gradlew
-gradle.properties
-gradle/
@@ -1,21 +1,4 @@
 include:
  - project: 'gitlab/gitlab'
    ref: 'main'
-    file: 'ci-templates/gradle_java.yml'
-
-deploy:
-  stage: deploy
-  tags:
-    - dind
-  script:
-    - echo "Building with gradle version ${BUILDVERSION}"
-    - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
-    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
-  artifacts:
-    reports:
-      dotenv: version.env
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
-    - if: $CI_COMMIT_BRANCH =~ /^release/
-    - if: $CI_COMMIT_TAG
+    file: 'ci-templates/gradle_java.yml'
@@ -1,16 +1,24 @@
 plugins {
-    `java-library`
-    `maven-publish`
+    java
    pmd
    checkstyle
    jacoco
 }

 group = "com.knecon.fforesight"
+version = "0.1-SNAPSHOT"

 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17

+tasks.jacocoTestReport {
+    reports {
+        xml.required.set(false)
+        csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
+    }
+}
+
 tasks.pmdMain {
    pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
 }
@@ -21,9 +29,6 @@ tasks.pmdTest {

 tasks.named<Test>("test") {
    useJUnitPlatform()
-    reports {
-        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
-    }
 }

 tasks.test {
@@ -35,38 +40,14 @@ tasks.jacocoTestReport {
    reports {
        xml.required.set(true)
        csv.required.set(false)
-        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
    }
 }

-allprojects {
-    publishing {
-        publications {
-            create<MavenPublication>(name) {
-                from(components["java"])
-            }
-        }
-        repositories {
-            maven {
-                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
-                credentials {
-                    username = providers.gradleProperty("mavenUser").getOrNull();
-                    password = providers.gradleProperty("mavenPassword").getOrNull();
-                }
-            }
-        }
-    }
-}
-
-java {
-    withJavadocJar()
-}
-
 repositories {
    mavenLocal()
    mavenCentral()
    maven {
-        url = uri("https://nexus.knecon.com/repository/gindev/")
+        url = uri("https://nexus.knecon.com/repository/gindev/");
        credentials {
            username = providers.gradleProperty("mavenUser").getOrNull();
            password = providers.gradleProperty("mavenPassword").getOrNull();
@@ -12,6 +12,7 @@
        <exclude name="DataflowAnomalyAnalysis"/>
        <exclude name="MissingSerialVersionUID"/>
        <exclude name="NullAssignment"/>
+        <exclude name="BeanMembersShouldSerialize"/>
        <exclude name="AvoidLiteralsInIfCondition"/>
        <exclude name="AvoidDuplicateLiterals"/>
        <exclude name="AvoidFieldNameMatchingMethodName"/>
@@ -13,11 +13,11 @@
        <exclude name="DataflowAnomalyAnalysis"/>
        <exclude name="MissingSerialVersionUID"/>
        <exclude name="NullAssignment"/>
+        <exclude name="BeanMembersShouldSerialize"/>
        <exclude name="AvoidLiteralsInIfCondition"/>
        <exclude name="AvoidDuplicateLiterals"/>
        <exclude name="AvoidFieldNameMatchingMethodName"/>
        <exclude name="AvoidFieldNameMatchingTypeName"/>
-        <exclude name="TestClassWithoutTestCases"/>
    </rule>

 </ruleset>
@@ -1 +1 @@
-version = 0.1-SNAPSHOT
+version = 0.1.0
@@ -1,6 +1,6 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.2.2"
+    id("io.freefair.lombok") version "8.1.0"
 }

 description = "layoutparser-service-internal-api"
@@ -1,7 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -23,40 +21,6 @@ public class DocumentStructure {
    EntryData root;


-    public static class TableProperties {
-
-        public static final String NUMBER_OF_ROWS = "numberOfRows";
-        public static final String NUMBER_OF_COLS = "numberOfCols";
-
-    }
-
-    public static class ImageProperties {
-
-        public static final String TRANSPARENT = "transparent";
-        public static final String IMAGE_TYPE = "imageType";
-        public static final String POSITION = "position";
-        public static final String ID = "id";
-
-    }
-
-    public static class TableCellProperties {
-
-        public static final String B_BOX = "bBox";
-        public static final String ROW = "row";
-        public static final String COL = "col";
-        public static final String HEADER = "header";
-
-    }
-
-    public static final String RECTANGLE_DELIMITER = ";";
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-
    public EntryData get(List<Integer> tocId) {

        if (tocId.isEmpty()) {
@@ -102,7 +66,6 @@ public class DocumentStructure {
        Map<String, String> properties;
        List<EntryData> children;

-
        @Override
        public String toString() {

@@ -4,22 +4,20 @@ import java.util.Map;
 import java.util.Optional;

 import lombok.Builder;
-import lombok.NonNull;
+

@Builder
 public record LayoutParsingRequest(
-        @NonNull LayoutParsingType layoutParsingType,
+        LayoutParsingType layoutParsingType,
        Map<String, String> identifier,
-        @NonNull String originFileStorageId,
+        String originFileStorageId,
        Optional<String> tablesFileStorageId,
        Optional<String> imagesFileStorageId,
-        @NonNull String structureFileStorageId,
+        String structureFileStorageId,
        String researchDocumentStorageId,
-        @NonNull String textBlockFileStorageId,
-        @NonNull String positionBlockFileStorageId,
-        @NonNull String pageFileStorageId,
-        @NonNull String simplifiedTextStorageId,
-        @NonNull String viewerDocumentStorageId,
-        @NonNull String sectionGridStorageId) {
+        String textBlockFileStorageId,
+        String positionBlockFileStorageId,
+        String pageFileStorageId,
+        String sectionGridStorageId) {

 }
@@ -1,26 +1,21 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.2.2"
+    id("io.freefair.lombok") version "8.1.0"
 }

-description = "layoutparser-service-processor"
-
-val jacksonVersion = "2.15.2"
-val pdfBoxVersion = "3.0.0-RC1"
-
 dependencies {
    implementation(project(":layoutparser-service-internal-api"))

-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.138.0") {
-        exclude("org.springframework.boot", "spring-boot-starter-security")
-        exclude("org.springframework.boot", "spring-boot-starter-validation")
-    }
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.36.0")
+    implementation("com.iqser.red.commons:storage-commons:2.1.0")

-    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
-    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
-    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
-    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-web:3.1.2")
+    implementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
+    implementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
+    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
+    implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
 }
+
+description = "layoutparser-service-processor"
@@ -2,57 +2,31 @@ package com.knecon.fforesight.service.layoutparser.processor;

 import static java.lang.String.format;

-import java.awt.geom.Rectangle2D;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.mapper.taas.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
-import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
-import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
-import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;

 import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@@ -63,19 +37,12 @@ public class LayoutParsingPipeline {
    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
    private final CvTableParsingAdapter cvTableParsingAdapter;
    private final LayoutParsingStorageService layoutParsingStorageService;
+    private final PdfParsingService pdfParsingService;
    private final SectionsBuilderService sectionsBuilderService;
    private final SectionGridCreatorService sectionGridCreatorService;
    private final TaasClassificationService taasClassificationService;
    private final RedactManagerClassificationService redactManagerClassificationService;
    private final DocuMineClassificationService docuMineClassificationService;
-    private final SimplifiedSectionTextService simplifiedSectionTextService;
-    private final BodyTextFrameService bodyTextFrameService;
-    private final RulingCleaningService rulingCleaningService;
-    private final TableExtractionService tableExtractionService;
-    private final TaasBlockificationService taasBlockificationService;
-    private final DocuMineBlockificationService docuMineBlockificationService;
-    private final RedactManagerBlockificationService redactManagerBlockificationService;
-    private final ViewerDocumentService viewerDocumentService;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -85,27 +52,19 @@ public class LayoutParsingPipeline {
        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
            }

            TableServiceResponse tableServiceResponse = new TableServiceResponse();
            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
            }

-            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
-            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
-
+            Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
            int numberOfPages = originDocument.getNumberOfPages();

            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
-
-            try (var out = new ByteArrayOutputStream()) {
-                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out);
-                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
-            }

            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
@@ -126,77 +85,15 @@ public class LayoutParsingPipeline {
    }


-    @SneakyThrows
-    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
-                                              PDDocument originDocument,
-                                              ImageServiceResponse imageServiceResponse,
-                                              TableServiceResponse tableServiceResponse) {
+    public Document parseLayout(LayoutParsingType layoutParsingType,
+                                PDDocument originDocument,
+                                ImageServiceResponse imageServiceResponse,
+                                TableServiceResponse tableServiceResponse) {

-        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
-        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
-
-        ClassificationDocument classificationDocument = new ClassificationDocument();
-        List<ClassificationPage> classificationPages = new ArrayList<>();
-
-        originDocument.setAllSecurityToBeRemoved(true);
-        long pageCount = originDocument.getNumberOfPages();
-
-        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-            classificationDocument.setPages(classificationPages);
-            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-            PDPage pdPage = originDocument.getPage(pageNumber - 1);
-            stripper.setPageNumber(pageNumber);
-            stripper.setStartPage(pageNumber);
-            stripper.setEndPage(pageNumber);
-            stripper.setPdpage(pdPage);
-            if (layoutParsingType.equals(LayoutParsingType.DOCUMINE)) {
-                stripper.setSortByPosition(true);
-            }
-            stripper.getText(originDocument);
-
-            PDRectangle pdr = pdPage.getMediaBox();
-
-            int rotation = pdPage.getRotation();
-            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-            PDRectangle cropbox = pdPage.getCropBox();
-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                    stripper.getRulings(),
-                    stripper.getMinCharWidth(),
-                    stripper.getMaxCharHeight());
-
-            ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER ->
-                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case TAAS ->
-                        taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-                case DOCUMINE ->
-                        docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-            };
-            classificationPage.setCleanRulings(cleanRulings);
-            classificationPage.setRotation(rotation);
-            classificationPage.setLandscape(isLandscape);
-            classificationPage.setPageNumber(pageNumber);
-            classificationPage.setPageWidth(cropbox.getWidth());
-            classificationPage.setPageHeight(cropbox.getHeight());
-
-            // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
-            classificationPage.setMarkedContentBboxPerType(convertMarkedContents(stripper.getMarkedContents()));
-
-            // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-            if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-                classificationPage.setImages(pdfImages.get(pageNumber));
-                imageServiceResponseAdapter.findOcr(classificationPage);
-            }
-
-            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
-            buildPageStatistics(classificationPage);
-            increaseDocumentStatistics(classificationPage, classificationDocument);
-
-            classificationPages.add(classificationPage);
-        }
-
-        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
+        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
+                originDocument,
+                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
+                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));

        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
@@ -207,45 +104,40 @@ public class LayoutParsingPipeline {
        sectionsBuilderService.buildSections(classificationDocument);
        sectionsBuilderService.addImagesToSections(classificationDocument);

-        return classificationDocument;
+        return DocumentGraphFactory.buildDocumentGraph(classificationDocument);
    }


-    private Map<String, Rectangle2D> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {
-        Map<String, Rectangle2D> markedContentBboxes = new HashMap<>();
-        markedContentBboxes.put(MarkedContentUtils.HEADER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.HEADER));
-        markedContentBboxes.put(MarkedContentUtils.FOOTER, MarkedContentUtils.getMarkedContentBboxPerLine(pdMarkedContents, MarkedContentUtils.FOOTER));
-        return markedContentBboxes;
-    }
+    public Document parseLayoutWithTimer(LayoutParsingType layoutParsingType,
+                                         PDDocument originDocument,
+                                         ImageServiceResponse imageServiceResponse,
+                                         TableServiceResponse tableServiceResponse) {

-    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
+        long start = System.currentTimeMillis();

-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
+                originDocument,
+                cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
+                imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
+
+        System.out.printf("parsed %d ms", System.currentTimeMillis() - start);
+
+        start = System.currentTimeMillis();
+        switch (layoutParsingType) {
+            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
+            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }
-        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
-        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
-        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
-    }
+        System.out.printf(", classified %d ms", System.currentTimeMillis() - start);

+        start = System.currentTimeMillis();
+        sectionsBuilderService.buildSections(classificationDocument);
+        System.out.printf(", sections built %d ms", System.currentTimeMillis() - start);

-    private void buildPageStatistics(ClassificationPage classificationPage) {
-
-        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
-        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
-            if (textBlock instanceof TextPageBlock) {
-                if (((TextPageBlock) textBlock).getSequences() == null) {
-                    continue;
-                }
-                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
-                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
-                    classificationPage.getFontCounter().add(word.getFont());
-                    classificationPage.getFontSizeCounter().add(word.getFontSize());
-                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
-                }
-            }
-        }
-
+        start = System.currentTimeMillis();
+        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+        System.out.printf(", graph constructed %d ms", System.currentTimeMillis() - start);
+        return document;
    }

 }
@@ -1,15 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor;

-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;

 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
@@ -20,16 +15,18 @@ import org.springframework.stereotype.Service;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

 import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@@ -43,11 +40,10 @@ public class LayoutParsingStorageService {

    public PDDocument getOriginFile(String storageId) throws IOException {

-        try (var originDocumentInputStream = getObject(storageId)) {
+        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
            File tempFile = createTempFile("document", ".pdf");
            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
-                originDocumentInputStream.close();
            }
            return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
        }
@@ -56,22 +52,19 @@ public class LayoutParsingStorageService {

    public ImageServiceResponse getImagesFile(String storageId) throws IOException {

-        try (InputStream inputStream = getObject(storageId)) {
+        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {

-            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
-            inputStream.close();
-            return imageServiceResponse;
+            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
        }
    }


    public TableServiceResponse getTablesFile(String storageId) throws IOException {

-        try (var tableClassificationStream = getObject(storageId)) {
+        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+
+            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);

-            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
-            tableClassificationStream.close();
-            return tableServiceResponse;
        }
    }

@@ -97,6 +90,26 @@ public class LayoutParsingStorageService {
    }


+    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
+
+        DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class);
+        DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
+                layoutParsingRequest.textBlockFileStorageId(),
+                DocumentTextData[].class);
+        DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
+                layoutParsingRequest.positionBlockFileStorageId(),
+                DocumentPositionData[].class);
+        DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class);
+
+        return DocumentData.builder()
+                .documentStructure(tableOfContentsData)
+                .documentPositions(atomicPositionBlockData)
+                .documentTextData(documentTextDataBlockData)
+                .documentPages(documentPageData)
+                .build();
+    }
+
+
    private File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {

        File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();
@@ -121,30 +134,4 @@ public class LayoutParsingStorageService {
        }
    }

-
-    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
-
-        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
-    }
-
-
-    @SneakyThrows
-    private InputStream getObject(String storageId) {
-
-        File tempFile = File.createTempFile("temp", ".data");
-        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
-        Path path = Paths.get(tempFile.getPath());
-        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
-    }
-
-
-    @SneakyThrows
-    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {
-
-        try (var in = new ByteArrayInputStream(out.toByteArray())) {
-
-            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
-        }
-    }
-
 }
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+package com.knecon.fforesight.service.layoutparser.processor.adapter;

 import java.util.ArrayList;
 import java.util.Collection;
@@ -8,8 +8,8 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;

 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+package com.knecon.fforesight.service.layoutparser.processor.adapter;

 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
@@ -9,10 +9,10 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;

 import lombok.RequiredArgsConstructor;

@@ -0,0 +1,14 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.Data;
+
+@Data
+public class Classification {
+
+    private Map<String, Float> probabilities = new HashMap<>();
+    private String label;
+
+}
@@ -0,0 +1,11 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import lombok.Data;
+
+@Data
+public class FilterGeometry {
+
+    private ImageSize imageSize;
+    private ImageFormat imageFormat;
+
+}
@@ -0,0 +1,12 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import lombok.Data;
+
+@Data
+public class Filters {
+
+    private FilterGeometry geometry;
+    private Probability probability;
+    private boolean allPassed;
+
+}
@@ -0,0 +1,11 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import lombok.Data;
+
+@Data
+public class Geometry {
+
+    private float width;
+    private float height;
+
+}
@@ -0,0 +1,12 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import lombok.Data;
+
+@Data
+public class ImageFormat {
+
+    private float quotient;
+    private boolean tooTall;
+    private boolean tooWide;
+
+}
@@ -0,0 +1,14 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import lombok.Data;
+
+@Data
+public class ImageMetadata {
+
+    private Classification classification;
+    private Position position;
+    private Geometry geometry;
+    private Filters filters;
+    private boolean alpha;
+
+}
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;

 import java.util.ArrayList;
 import java.util.List;
@@ -6,15 +6,9 @@ import java.util.List;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;

-import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
-import lombok.NoArgsConstructor;

@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
 public class ImageServiceResponse {

    private String dossierId;
@@ -0,0 +1,12 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import lombok.Data;
+
+@Data
+public class ImageSize {
+
+    private float quotient;
+    private boolean tooLarge;
+    private boolean tooSmall;
+
+}
@@ -0,0 +1,14 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import lombok.Data;
+
+@Data
+public class Position {
+
+    private float x1;
+    private float x2;
+    private float y1;
+    private float y2;
+    private int pageNumber;
+
+}
@@ -0,0 +1,10 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
+
+import lombok.Data;
+
+@Data
+public class Probability {
+
+    private boolean unconfident;
+
+}
@@ -0,0 +1,13 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
+
+import lombok.Data;
+
+@Data
+public class PageInfo {
+
+    private int number;
+    private int rotation;
+    private float width;
+    private float height;
+
+}
@@ -1,14 +1,14 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;

@Data
@Builder
-@NoArgsConstructor
@AllArgsConstructor
+@RequiredArgsConstructor
 public class PdfTableCell {

    private float x0;
@@ -1,14 +1,10 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;

-import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.NoArgsConstructor;

@Data
@Builder
-@NoArgsConstructor
-@AllArgsConstructor
 public class TableCells {

    private float x0;
@@ -0,0 +1,14 @@
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class TableData {
+
+    private PageInfo pageInfo;
+    private List<TableCells> tableCells = new ArrayList<>();
+
+}
@@ -1,17 +1,11 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;

 import java.util.ArrayList;
 import java.util.List;

-import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
-import lombok.NoArgsConstructor;

@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
 public class TableServiceResponse {

    private String dossierId;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.factory;
+package com.knecon.fforesight.service.layoutparser.processor.factory;

 import static java.lang.String.format;
 import static java.util.stream.Collectors.groupingBy;
@@ -20,17 +20,17 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Header;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Footer;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Header;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Headline;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Paragraph;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

@@ -1,10 +1,10 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.factory;
+package com.knecon.fforesight.service.layoutparser.processor.factory;

 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;

 import lombok.AccessLevel;
 import lombok.Builder;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.factory;
+package com.knecon.fforesight.service.layoutparser.processor.factory;

 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
@@ -11,7 +11,7 @@ import java.util.Objects;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;

 import lombok.experimental.UtilityClass;

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.factory;
+package com.knecon.fforesight.service.layoutparser.processor.factory;

 import static java.lang.String.format;
 import static java.util.Collections.emptyList;
@@ -14,9 +14,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TableMergingUtility;

 import lombok.experimental.UtilityClass;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.factory;
+package com.knecon.fforesight.service.layoutparser.processor.factory;

 import static java.util.Collections.emptyList;

@@ -11,12 +11,12 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import lombok.experimental.UtilityClass;
@@ -32,12 +32,7 @@ public class TableNodeFactory {
        setPageNumberInCells(tablesToMerge);
        Set<Page> pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet());
        List<List<Cell>> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList();
-
-        Table table = Table.builder()
-                .documentTree(context.getDocumentTree())
-                .numberOfCols(mergedRows.isEmpty() ? 0 : mergedRows.get(0).size())
-                .numberOfRows(mergedRows.size())
-                .build();
+        Table table = Table.builder().documentTree(context.getDocumentTree()).numberOfCols(mergedRows.get(0).size()).numberOfRows(mergedRows.size()).build();

        pages.forEach(page -> addTableToPage(page, parentNode, table));

@@ -1,11 +1,11 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.factory;
+package com.knecon.fforesight.service.layoutparser.processor.factory;

 import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;

 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph;
+package com.knecon.fforesight.service.layoutparser.processor.graph;

 import static java.lang.String.format;

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph;
+package com.knecon.fforesight.service.layoutparser.processor.graph;

 import static java.lang.String.format;

@@ -8,13 +8,13 @@ import java.util.List;
 import java.util.stream.Stream;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.graph.entity;

 public enum EntityType {
    ENTITY,
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.graph.entity;

 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
@@ -11,9 +11,9 @@ import java.util.Map;
 import java.util.Set;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;

 import lombok.AccessLevel;
@@ -1,9 +1,9 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
+package com.knecon.fforesight.service.layoutparser.processor.graph.entity;

 import java.awt.geom.Rectangle2D;
 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -12,10 +12,10 @@ import java.util.stream.Stream;

 import com.amazonaws.services.kms.model.NotFoundException;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -0,0 +1,5 @@
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
+
+public interface GenericSemanticNode extends SemanticNode {
+
+}
@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -9,10 +9,10 @@ import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.util.Locale;

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.util.HashSet;
 import java.util.LinkedList;
@@ -6,9 +6,9 @@ import java.util.List;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,14 +1,14 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import static java.lang.String.format;

@@ -13,12 +13,12 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.EntityType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.EntityType;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 public interface SemanticNode {
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import static java.lang.String.format;

@@ -11,10 +11,10 @@ import java.util.stream.IntStream;
 import java.util.stream.Stream;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;

 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
@@ -8,10 +8,10 @@ import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;

 import static java.lang.String.format;

@@ -13,9 +13,9 @@ import java.util.Map;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.AccessLevel;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;

 import static java.lang.String.format;

@@ -11,8 +11,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;

-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;

 import lombok.AccessLevel;
 import lombok.Data;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;

 import static java.lang.String.format;

@@ -10,8 +10,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;

-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;

 public interface TextBlock extends CharSequence {

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
+package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;

 import java.util.Set;
 import java.util.function.BiConsumer;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
+package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;

 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
@@ -12,14 +12,14 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;

 import lombok.experimental.UtilityClass;

@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
+package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;

 import java.util.Arrays;
 import java.util.HashSet;
@@ -12,21 +12,21 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Header;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Footer;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Header;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Headline;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Paragraph;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;

 import lombok.experimental.UtilityClass;

@@ -0,0 +1,112 @@
+package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
+
+public class PropertiesMapper {
+
+    public static final String TRANSPARENT = "transparent";
+    public static final String IMAGE_TYPE = "imageType";
+    public static final String POSITION = "position";
+    public static final String ROW = "row";
+    public static final String COL = "col";
+    public static final String HEADER = "header";
+    public static final String B_BOX = "bBox";
+    public static final String NUMBER_OF_ROWS = "numberOfRows";
+    public static final String NUMBER_OF_COLS = "numberOfCols";
+
+
+    public static Map<String, String> buildImageProperties(Image image) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put(IMAGE_TYPE, image.getImageType().toString());
+        properties.put(TRANSPARENT, String.valueOf(image.isTransparent()));
+        properties.put(POSITION, RectangleTransformations.toString(image.getPosition()));
+        return properties;
+    }
+
+
+    public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put(ROW, String.valueOf(tableCell.getRow()));
+        properties.put(COL, String.valueOf(tableCell.getCol()));
+        properties.put(HEADER, String.valueOf(tableCell.isHeader()));
+
+        if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
+            throw new IllegalArgumentException("TableCell can only occur on a single page!");
+        }
+        String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
+        properties.put(B_BOX, bBoxString);
+
+        return properties;
+    }
+
+
+    public static Map<String, String> buildTableProperties(Table table) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put(NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
+        properties.put(NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
+        return properties;
+    }
+
+
+    public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
+
+        builder.imageType(parseImageType(properties.get(IMAGE_TYPE)));
+        builder.transparent(Boolean.parseBoolean(properties.get(TRANSPARENT)));
+        builder.position(parseRectangle2D(properties.get(POSITION)));
+    }
+
+
+    public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
+
+        builder.row(Integer.parseInt(properties.get(ROW)));
+        builder.col(Integer.parseInt(properties.get(COL)));
+        builder.header(Boolean.parseBoolean(properties.get(HEADER)));
+        builder.bBox(parseRectangle2D(properties.get(B_BOX)));
+    }
+
+
+    public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
+
+        builder.numberOfRows(Integer.parseInt(properties.get(NUMBER_OF_ROWS)));
+        builder.numberOfCols(Integer.parseInt(properties.get(NUMBER_OF_COLS)));
+    }
+
+
+    private static ImageType parseImageType(String imageType) {
+
+        return switch (imageType) {
+            case "LOGO" -> ImageType.LOGO;
+            case "FORMULA" -> ImageType.FORMULA;
+            case "SIGNATURE" -> ImageType.SIGNATURE;
+            case "OCR" -> ImageType.OCR;
+            default -> ImageType.OTHER;
+        };
+    }
+
+
+    public static String toString(Rectangle2D rectangle2D) {
+
+        return String.format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
+    }
+
+
+    public static Rectangle2D parseRectangle2D(String bBox) {
+
+        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
+    }
+
+}
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
+package com.knecon.fforesight.service.layoutparser.processor.mapper.taas;

 import java.awt.geom.Rectangle2D;
 import java.util.List;
@@ -13,13 +13,13 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.Researc
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.RowData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.StructureObject;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.TableData;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;

 public class TaasDocumentDataMapper {

@@ -1,20 +1,15 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;

-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;

 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
-import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;

@Data
@RequiredArgsConstructor
@@ -40,8 +35,4 @@ public class ClassificationPage {
    private float pageWidth;
    private float pageHeight;

-    CleanRulings cleanRulings;
-
-    private Map<String, Rectangle2D> markedContentBboxPerType = new HashMap<>();
-
 }
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.awt.geom.Rectangle2D;
 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

 import lombok.AllArgsConstructor;
@@ -18,5 +17,5 @@ public class PageContents {
    List<TextPositionSequence> sortedTextPositionSequences;
    Rectangle2D cropBox;
    Rectangle2D mediaBox;
-    List<Ruling> rulings;
+
 }
@@ -1,5 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
-
-public interface GenericSemanticNode extends SemanticNode {
-
-}
@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.image;

 import java.awt.geom.Rectangle2D;

-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;

 import lombok.Data;
 import lombok.NonNull;
@@ -28,7 +28,6 @@ public class Ruling extends Line2D.Float {
        super(p1, p2);
    }

-
    public Ruling straightenVertical() {

        double y1 = Math.min(getY1(), getY2());
@@ -37,7 +36,6 @@ public class Ruling extends Line2D.Float {
        return new Ruling(new Point2D.Double(x, y1), new Point2D.Double(x, y2));
    }

-
    public Ruling straightenHorizontal() {

        double x1 = Math.min(getX1(), getX2());
@@ -446,16 +444,6 @@ public class Ruling extends Line2D.Float {
    }


-    public boolean almostMatches(Ruling ruling) {
-
-        final float TOLERANCE = 1;
-        return Math.abs(ruling.getX1() - x1) < TOLERANCE &&//
-                Math.abs(ruling.getY1() - y1) < TOLERANCE &&//
-                Math.abs(ruling.getX2() - x2) < TOLERANCE &&//
-                Math.abs(ruling.getY2() - y2) < TOLERANCE;
-    }
-
-
    private enum SOType {
        VERTICAL,
        HRIGHT,
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import java.util.ArrayList;
 import java.util.List;
@@ -73,7 +73,6 @@ public class TextPageBlock extends AbstractPageBlock {

        return sequences.get(0).getPageWidth();
    }
-    

    public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {

@@ -1,8 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
-
-import java.awt.Color;
-import java.awt.geom.Line2D;
-
-public record ColoredLine(Line2D line, Color color) {
-
-}
@@ -1,8 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
-
-import java.awt.Color;
-import java.awt.geom.Rectangle2D;
-
-public record ColoredRectangle(Rectangle2D rectangle2D, Color color) {
-
-}
@@ -1,8 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
-
-import java.awt.Color;
-import java.awt.geom.Rectangle2D;
-
-public record FilledRectangle(Rectangle2D rectangle2D, Color color, float alpha) {
-
-}
@@ -1,27 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class LayoutGrid {
-
-    int numberOfPages;
-    List<VisualizationsOnPage> visualizationsPerPages;
-
-
-    public LayoutGrid(int numberOfPages) {
-
-        this.numberOfPages = numberOfPages;
-        this.visualizationsPerPages = new ArrayList<>(numberOfPages);
-        for (int i = 0; i < numberOfPages; i++) {
-            this.visualizationsPerPages.add(VisualizationsOnPage.builder().pageNumber(i).build());
-        }
-    }
-
-}
@@ -1,7 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
-
-import java.awt.geom.Point2D;
-
-public record PlacedText(String text, Point2D lineStart) {
-
-}
@@ -1,26 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import lombok.AccessLevel;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@Builder
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class VisualizationsOnPage {
-
-    int pageNumber;
-    @Builder.Default
-    List<PlacedText> placedTexts = new LinkedList<>();
-    @Builder.Default
-    List<ColoredLine> coloredLines = new LinkedList<>();
-    @Builder.Default
-    List<ColoredRectangle> coloredRectangles = new LinkedList<>();
-    @Builder.Default
-    List<FilledRectangle> filledRectangles = new LinkedList<>();
-
-}
@@ -14,7 +14,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
+package com.knecon.fforesight.service.layoutparser.processor.parsing;

 import java.io.IOException;
 import java.io.InputStream;
@@ -1,18 +1,32 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
+package com.knecon.fforesight.service.layoutparser.processor.parsing;
+
+import java.awt.color.CMMException;
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.*;
-import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
-import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
-import org.apache.pdfbox.contentstream.operator.state.*;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
+import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
+import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
+import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
+import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
+import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
+import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
+import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
+import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
+import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSNumber;
@@ -21,11 +35,14 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.text.TextPosition;

-import java.awt.color.CMMException;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;

@Getter
@Slf4j
@@ -41,7 +58,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
    private int minCharHeight;
    private int maxCharHeight;

-
    private float path_x;
    private float path_y;

@@ -72,12 +88,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        this.addOperator(new SetNonStrokingColorN());
        this.addOperator(new SetFontAndSize());
        this.addOperator(new SetLineWidth());
-
-
-        addOperator(new BeginMarkedContentSequenceWithProperties());
-//        addOperator(new BeginMarkedContentSequence());
-        addOperator(new EndMarkedContentSequence());
-
    }


@@ -88,7 +98,6 @@ public class PDFLinesTextStripper extends PDFTextStripper {

        //move
        switch (operation) {
-
            case OperatorName.MOVE_TO:
                if (arguments.size() == 2) {
                    Point2D.Float pos = transformPosition(floatValue(arguments.get(0)), floatValue(arguments.get(1)));
@@ -210,11 +219,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            // This is a quick and dirt hack
            // Happens for file 216.pdf
            log.debug(e.getMessage());
-            var result = true;
-            for (var component : color.getComponents()) {
-                result = result && component == 0;
-            }
-            return result;
+            return color.getComponents()[0] == 0 && color.getComponents()[1] == 0 && color.getComponents()[2] == 0 && color.getComponents()[1] == 1;
        }
    }

@@ -339,4 +344,3 @@ public class PDFLinesTextStripper extends PDFTextStripper {

 }

-
@@ -14,7 +14,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
+package com.knecon.fforesight.service.layoutparser.processor.parsing;

 import java.io.BufferedInputStream;
 import java.io.IOException;
@@ -25,20 +25,27 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.regex.Pattern;

-import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.pdfbox.cos.COSDictionary;
-import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageTree;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
-import org.apache.pdfbox.pdmodel.graphics.PDXObject;
 import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
 import org.apache.pdfbox.pdmodel.interactive.pagenavigation.PDThreadBead;
 import org.apache.pdfbox.text.TextPosition;
@@ -56,10 +63,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    private static float defaultDropThreshold = 2.5f;
    private static final boolean useCustomQuickSort;

-    @Getter
-    protected final List<PDMarkedContent> markedContents = new ArrayList<>();
-    protected final Deque<PDMarkedContent> currentMarkedContents = new ArrayDeque<>();
-
    private static final Log LOG = LogFactory.getLog(PDFTextStripper.class);

    // enable the ability to set the default indent/drop thresholds
@@ -193,45 +196,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    }


-
-    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
-    {
-        PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
-        if (this.currentMarkedContents.isEmpty())
-        {
-            this.markedContents.add(markedContent);
-        }
-        else
-        {
-            PDMarkedContent currentMarkedContent =
-                    this.currentMarkedContents.peek();
-            if (currentMarkedContent != null)
-            {
-                currentMarkedContent.addMarkedContent(markedContent);
-            }
-        }
-        this.currentMarkedContents.push(markedContent);
-    }
-
-    @Override
-    public void endMarkedContentSequence()
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
-            this.currentMarkedContents.pop();
-        }
-    }
-
-
-    public void xobject(PDXObject xobject)
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
-            this.currentMarkedContents.peek().addXObject(xobject);
-        }
-    }
-
-
    /**
     * This will return the text of a document. See writeText. <br>
     * NOTE: The document must not be encrypted when coming into this method.
@@ -913,12 +877,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    textList.add(text);
                }
            }
-            if (!this.currentMarkedContents.isEmpty())
-            {
-                this.currentMarkedContents.peek().addText(text);
-            }
        }
-
    }


@@ -2144,7 +2103,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            this.isHangingIndent = true;
        }

-
    }

 }
@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class Classification {
-
-    private Map<String, Float> probabilities = new HashMap<>();
-    private String label;
-
-}
@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class FilterGeometry {
-
-    private ImageSize imageSize;
-    private ImageFormat imageFormat;
-
-}
@@ -1,18 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class Filters {
-
-    private FilterGeometry geometry;
-    private Probability probability;
-    private boolean allPassed;
-
-}
@@ -1,17 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class Geometry {
-
-    private float width;
-    private float height;
-
-}
@@ -1,18 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImageFormat {
-
-    private float quotient;
-    private boolean tooTall;
-    private boolean tooWide;
-
-}
@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImageMetadata {
-
-    private Classification classification;
-    private Position position;
-    private Geometry geometry;
-    private Filters filters;
-    private boolean alpha;
-
-}
@@ -1,18 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImageSize {
-
-    private float quotient;
-    private boolean tooLarge;
-    private boolean tooSmall;
-
-}
@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class Position {
-
-    private float x1;
-    private float x2;
-    private float y1;
-    private float y2;
-    private int pageNumber;
-
-}
@@ -1,16 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class Probability {
-
-    private boolean unconfident;
-
-}
@@ -1,19 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class PageInfo {
-
-    private int number;
-    private int rotation;
-    private float width;
-    private float height;
-
-}
@@ -1,20 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class TableData {
-
-    private PageInfo pageInfo;
-    private List<TableCells> tableCells = new ArrayList<>();
-
-}
@@ -1,4 +1,7 @@
-package com.knecon.fforesight.service.layoutparser.server.queue;
+package com.knecon.fforesight.service.layoutparser.processor.queue;
+
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;

 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
@@ -9,9 +12,7 @@ import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;

 import lombok.RequiredArgsConstructor;
@@ -29,15 +30,12 @@ public class MessageHandler {


    @RabbitHandler
-    @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)
+    @RabbitListener(queues = LAYOUT_PARSING_REQUEST_QUEUE)
    @SneakyThrows
    public void receiveLayoutParsingRequest(Message message) {

        LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);

-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS) && layoutParsingRequest.researchDocumentStorageId() == null) {
-            throw new IllegalArgumentException("ResearchDocumentDataStorageId is null!");
-        }
        log.info("Layout parsing request received {}", layoutParsingRequest.identifier());
        if (message.getMessageProperties().isRedelivered()) {
            throw new AmqpRejectAndDontRequeueException(String.format("Error during last layout parsing of request with identifier: %s, do not retry.",
@@ -51,7 +49,7 @@ public class MessageHandler {

    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {

-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
+        rabbitTemplate.convertAndSend(LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
    }

 }
@@ -0,0 +1,40 @@
+package com.knecon.fforesight.service.layoutparser.processor.queue;
+
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
+import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import lombok.RequiredArgsConstructor;
+
+@Configuration
+@RequiredArgsConstructor
+public class MessagingConfiguration {
+
+    @Bean
+    public Queue layoutparsingRequestQueue() {
+
+        return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    @Bean
+    public Queue layoutparsingResponseQueue() {
+
+        return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    @Bean
+    public Queue layoutparsingDLQ() {
+
+        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
+    }
+
+}
@@ -2,14 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor.services;

 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -21,64 +19,6 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
 public class BodyTextFrameService {


-    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {
-
-        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
-        Rectangle landscapeBodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), true, layoutParsingType);
-        for (ClassificationPage page : classificationDocument.getPages()) {
-            setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
-        }
-    }
-
-
-/*
-    private Rectangle calculateBodyTextFrameByRulings(List<ClassificationPage> pages) {
-
-        Map<ClassificationPage, List<Ruling>> potentialHeaderRulingsPerPage = new HashMap<>();
-        Map<ClassificationPage, List<Ruling>> potentialFooterRulingsPerPage = new HashMap<>();
-
-        for (var page : pages) {
-            potentialHeaderRulingsPerPage.put(page,
-                    page.getCleanRulings()
-                            .getHorizontal()
-                            .stream()
-                            .filter(ruling -> ruling.getY1() > page.getPageHeight() * 0.8)
-                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
-                            .toList());
-            potentialFooterRulingsPerPage.put(page,
-                    page.getCleanRulings()
-                            .getHorizontal()
-                            .stream()
-                            .filter(ruling -> ruling.getY1() < page.getPageHeight() * 0.2)
-                            .filter(ruling -> ruling.getWidth() > 0.6 * page.getPageWidth())
-                            .toList());
-        }
-
-        Optional<Ruling> headerRuling = potentialHeaderRulingsPerPage.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .filter(ruling -> potentialHeaderRulingsPerPage.values()
-                        .stream()
-                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
-                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
-                .min(Comparator.comparingDouble(Ruling::getY1));
-
-        Optional<Ruling> footerRuling = potentialFooterRulingsPerPage.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .filter(ruling -> potentialHeaderRulingsPerPage.values()
-                        .stream()
-                        .filter(rulingsPerPage -> rulingsPerPage.stream().anyMatch(ruling::almostMatches))
-                        .count() > pages.size() * RULING_THRESHOLD_FACTOR)
-                .max(Comparator.comparingDouble(Ruling::getY1));
-
-        double maxY = headerRuling.isPresent() ? headerRuling.get().y1 : pages.stream().mapToDouble(ClassificationPage::getPageHeight).max().orElse(Double.MAX_VALUE);
-        double minY = footerRuling.map(ruling -> ruling.y1).orElse(0F);
-        double maxX = pages.stream().mapToDouble(ClassificationPage::getPageWidth).max().orElse(Double.MAX_VALUE);
-
-        return new Rectangle(new Point((float) maxX, (float) maxY), (float) 0, (float) minY, -1);
-    }
-*/


    /**
@@ -94,7 +34,7 @@ public class BodyTextFrameService {
     * @param bodyTextFrame          frame that contains the main text on portrait pages
     * @param landscapeBodyTextFrame frame that contains the main text on landscape pages
     */
-    private void setBodyTextFrameAdjustedToPage(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
+    public void setBodyTextFrameAdjustedToPage(ClassificationPage page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {

        Rectangle textFrame = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;

@@ -129,10 +69,7 @@ public class BodyTextFrameService {
     * @param landscape               Calculate for landscape or portrait
     * @return Rectangle of the text frame
     */
-    private Rectangle calculateBodyTextFrame(List<ClassificationPage> pages,
-                                             FloatFrequencyCounter documentFontSizeCounter,
-                                             boolean landscape,
-                                             LayoutParsingType layoutParsingType) {
+    public Rectangle calculateBodyTextFrame(List<ClassificationPage> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape, LayoutParsingType layoutParsingType) {

        float approximateHeaderLineCount;
        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
@@ -157,14 +94,9 @@ public class BodyTextFrameService {
                        continue;
                    }

-                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
-                        continue;
-                    }
-
                    float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
-                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10) || !layoutParsingType.equals(
-                            LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount) {
+                    if (layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount && textBlock.getMaxY() >= page.getPageHeight() - (page.getPageHeight() / 10)
+                            || !layoutParsingType.equals(LayoutParsingType.DOCUMINE) && approxLineCount < approximateHeaderLineCount){
                        continue;
                    }

@@ -238,4 +170,4 @@ public class BodyTextFrameService {

    }

-}
+}
@@ -0,0 +1,151 @@
+package com.knecon.fforesight.service.layoutparser.processor.services;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class PdfParsingService {
+
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;
+
+
+    public ClassificationDocument parseDocument(LayoutParsingType layoutParsingType,
+                                                PDDocument originDocument,
+                                                Map<Integer, List<TableCells>> pdfTableCells,
+                                                Map<Integer, List<ClassifiedImage>> pdfImages) {
+
+        ClassificationDocument document = new ClassificationDocument();
+        List<ClassificationPage> classificationPages = new ArrayList<>();
+
+        originDocument.setAllSecurityToBeRemoved(true);
+        long pageCount = originDocument.getNumberOfPages();
+
+        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
+            parsePage(layoutParsingType, pdfImages, originDocument, pdfTableCells, document, classificationPages, pageNumber);
+        }
+
+        document.setPages(classificationPages);
+
+        return document;
+    }
+
+
+    @SneakyThrows
+    private void parsePage(LayoutParsingType layoutParsingType,
+                           Map<Integer, List<ClassifiedImage>> pdfImages,
+                           PDDocument pdDocument,
+                           Map<Integer, List<TableCells>> pdfTableCells,
+                           ClassificationDocument document,
+                           List<ClassificationPage> classificationPages,
+                           int pageNumber) {
+
+        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+        stripper.setPageNumber(pageNumber);
+        stripper.setStartPage(pageNumber);
+        stripper.setEndPage(pageNumber);
+        stripper.setPdpage(pdPage);
+        stripper.getText(pdDocument);
+
+        PDRectangle pdr = pdPage.getMediaBox();
+
+        int rotation = pdPage.getRotation();
+        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+        PDRectangle cropbox = pdPage.getCropBox();
+        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                stripper.getRulings(),
+                stripper.getMinCharWidth(),
+                stripper.getMaxCharHeight());
+
+        ClassificationPage classificationPage = switch (layoutParsingType) {
+            case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+        };
+
+        classificationPage.setRotation(rotation);
+        classificationPage.setLandscape(isLandscape);
+        classificationPage.setPageNumber(pageNumber);
+        classificationPage.setPageWidth(cropbox.getWidth());
+        classificationPage.setPageHeight(cropbox.getHeight());
+
+        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+            classificationPage.setImages(pdfImages.get(pageNumber));
+            imageServiceResponseAdapter.findOcr(classificationPage);
+        }
+
+        tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
+        buildPageStatistics(classificationPage);
+        increaseDocumentStatistics(classificationPage, document);
+
+        classificationPages.add(classificationPage);
+    }
+
+
+    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
+
+        if (!classificationPage.isLandscape()) {
+            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        }
+        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
+        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
+        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
+    }
+
+
+    private void buildPageStatistics(ClassificationPage classificationPage) {
+
+        // Collect all statistics for the classificationPage, except from blocks inside tables, as tables will always be added to BodyTextFrame.
+        for (AbstractPageBlock textBlock : classificationPage.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
+                    continue;
+                }
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
+                    classificationPage.getTextHeightCounter().add(word.getTextHeight());
+                    classificationPage.getFontCounter().add(word.getFont());
+                    classificationPage.getFontSizeCounter().add(word.getFontSize());
+                    classificationPage.getFontStyleCounter().add(word.getFontStyle());
+                }
+            }
+        }
+
+    }
+
+}
+
+
@@ -12,7 +12,7 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;
@@ -20,11 +20,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;

 import lombok.RequiredArgsConstructor;

@@ -1,26 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services;
-
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
-
-@Service
-public class SimplifiedSectionTextService {
-
-    public SimplifiedText toSimplifiedText(Document document) {
-
-        List<SimplifiedSectionText> simplifiedSectionTexts = document.getMainSections().stream().map(this::toSimplifiedSectionText).toList();
-        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build();
-    }
-
-
-    private SimplifiedSectionText toSimplifiedSectionText(Section section) {
-
-        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
-    }
-}
@@ -15,15 +15,15 @@ import org.springframework.core.io.ClassPathResource;

 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
+import com.knecon.fforesight.service.layoutparser.processor.parsing.PDFLinesTextStripper;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.experimental.UtilityClass;

@UtilityClass
-public class PageContentExtractor {
+public class TextPositionSequenceSorter {

-    public List<PageContents> getSortedPageContents(String filename) throws IOException {
+    public List<PageContents> getSortedTextPositionsWithPages(String filename) throws IOException {

        List<PageContents> textPositionSequencesPerPage = new LinkedList<>();
        try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
@@ -49,8 +49,7 @@ public class PageContentExtractor {

                    textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
                            RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
-                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox()),
-                            stripper.getRulings()));
+                            RectangleTransformations.toRectangle2D(pdPage.getMediaBox())));
                }
            }
        }
@@ -5,9 +5,6 @@ import static java.util.stream.Collectors.toSet;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

@@ -26,8 +23,6 @@ public class DocuMineBlockificationService {

    static final float THRESHOLD = 1f;

-    Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z ()-]{2,50}", Pattern.CASE_INSENSITIVE);
-

    /**
     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
@@ -48,6 +43,7 @@ public class DocuMineBlockificationService {
        float maxX = 0;
        float minY = 1000;
        float maxY = 0;
+
        TextPositionSequence prev = null;

        boolean wasSplitted = false;
@@ -64,10 +60,7 @@ public class DocuMineBlockificationService {
            boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle()
                    .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));

-            Matcher matcher = pattern.matcher(chunkWords.stream().collect(Collectors.joining(" ")).toString());
-            boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();
-
-            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap || startsOnSameX)) {
+            if (prev != null && (lineSeparation || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || negativeXGap)) {

                Orientation prevOrientation = null;
                if (!chunkBlockList1.isEmpty()) {
@@ -238,4 +231,3 @@ public class DocuMineBlockificationService {
    }

 }
-
@@ -5,36 +5,43 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

+
@Slf4j
@Service
@RequiredArgsConstructor
 public class DocuMineClassificationService {

-    private static final Pattern pattern = Pattern.compile("^(\\d{1,2}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern2 = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
+    private final BodyTextFrameService bodyTextFrameService;
+    private static final Pattern pattern = Pattern.compile("^(\\d{1,1}\\.?){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z\\[\\]\\-]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pattern2 = Pattern.compile(".*\\d{4}$", Pattern.CASE_INSENSITIVE);
    private static final Pattern pattern3 = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");


    public void classifyDocument(ClassificationDocument document) {

+        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.DOCUMINE);
+        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.DOCUMINE);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
+            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
            classifyPage(page, document, headlineFontSizes);
        }
    }
@@ -63,16 +70,12 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
-        ) {
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
-        ) {
+        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -112,4 +115,4 @@ public class DocuMineClassificationService {
        }
    }

-}
+}
@@ -3,14 +3,16 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -21,14 +23,19 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RedactManagerClassificationService {

+    private final BodyTextFrameService bodyTextFrameService;
+

    public void classifyDocument(ClassificationDocument document) {

+        Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false, LayoutParsingType.REDACT_MANAGER);
+        Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true, LayoutParsingType.REDACT_MANAGER);
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

        for (ClassificationPage page : document.getPages()) {
+            bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
            classifyPage(page, document, headlineFontSizes);
        }
    }
@@ -52,13 +59,11 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
--- a/Show More
+++ b/Show More