RED-7375 table extractor prototype

2024-02-01 08:23:53 +01:00 · 2024-01-30 11:59:18 +01:00 · 2024-01-17 16:58:53 +01:00 · 2024-01-15 09:43:23 +01:00 · 2023-12-01 13:26:16 +01:00 · 2023-11-29 10:06:05 +01:00
140 changed files with 991 additions and 3234 deletions
@@ -24,8 +24,6 @@ tasks.named<Test>("test") {
    reports {
        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
    }
-    minHeapSize = "512m"
-    maxHeapSize = "2048m"
 }

 tasks.test {
@@ -9,13 +9,12 @@
    </description>

    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
        <exclude name="AvoidLiteralsInIfCondition"/>
        <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="NullAssignment"/>
-        <exclude name="AssignmentInOperand"/>
-        <exclude name="BeanMembersShouldSerialize"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
    </rule>

-</ruleset>
-
+</ruleset>
@@ -10,14 +10,14 @@


    <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
        <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
        <exclude name="AvoidLiteralsInIfCondition"/>
        <exclude name="AvoidDuplicateLiterals"/>
        <exclude name="AvoidFieldNameMatchingMethodName"/>
-        <exclude name="NullAssignment"/>
-        <exclude name="AssignmentInOperand"/>
+        <exclude name="AvoidFieldNameMatchingTypeName"/>
        <exclude name="TestClassWithoutTestCases"/>
-        <exclude name="BeanMembersShouldSerialize"/>
    </rule>

-</ruleset>
+</ruleset>
@@ -1,6 +1,6 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.4"
+    id("io.freefair.lombok") version "8.2.2"
 }

 description = "layoutparser-service-internal-api"
@@ -1,7 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

-import java.io.Serializable;
-
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -14,7 +12,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Schema(description = "Object containing the complete document layout parsing information. It is split into 4 categories, structure, text, positions and pages: " + "The document tree structure of SemanticNodes such as Section, Paragraph, Headline, etc. " + "The text, which is stored as separate blocks of data. " + "The text positions, which are also stored as separate blocks. The Blocks are equal to the text blocks in length and order. " + "The page information.")
-public class DocumentData implements Serializable {
+public class DocumentData {

    @Schema(description = "Contains information about the document's pages.")
    DocumentPage[] documentPages;
@@ -1,7 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

-import java.io.Serializable;
-
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,7 +14,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@Schema(description = "Object containing information about the document's pages.")
-public class DocumentPage implements Serializable {
+public class DocumentPage {

    @Schema(description = "The page number, starting with 1.")
    int number;
@@ -1,7 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

-import java.io.Serializable;
-
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,7 +14,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@Schema(description = "Object containing text positional information of a specific text block. A document is split into multiple text blocks, which are supposed to be read in order. Every text block can only occur on a single page.")
-public class DocumentPositionData implements Serializable {
+public class DocumentPositionData {

    @Schema(description = "Identifier of the text block.")
    Long id;
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

 import java.awt.geom.Rectangle2D;
-import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -21,13 +20,14 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@Schema(description = "Object containing information about the parsed tree structure of the SemanticNodes, such as Section, Paragraph, Headline etc inside of the document.")
-public class DocumentStructure implements Serializable {
+public class DocumentStructure {

    @Schema(description = "The root EntryData represents the Document.")
    EntryData root;

+
    @Schema(description = "Object containing the extra field names, a table has in its properties field.")
-    public static class TableProperties implements Serializable {
+    public static class TableProperties {

        public static final String NUMBER_OF_ROWS = "numberOfRows";
        public static final String NUMBER_OF_COLS = "numberOfCols";
@@ -35,7 +35,7 @@ public class DocumentStructure implements Serializable {
    }

    @Schema(description = "Object containing the extra field names, an Image has in its properties field.")
-    public static class ImageProperties implements Serializable {
+    public static class ImageProperties {

        public static final String TRANSPARENT = "transparent";
        public static final String IMAGE_TYPE = "imageType";
@@ -45,7 +45,7 @@ public class DocumentStructure implements Serializable {
    }

    @Schema(description = "Object containing the extra field names, a table cell has in its properties field.")
-    public static class TableCellProperties implements Serializable {
+    public static class TableCellProperties {

        public static final String B_BOX = "bBox";
        public static final String ROW = "row";
@@ -56,7 +56,6 @@ public class DocumentStructure implements Serializable {

    public static final String RECTANGLE_DELIMITER = ";";

-
    public static Rectangle2D parseRectangle2D(String bBox) {

        List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
@@ -101,7 +100,7 @@ public class DocumentStructure implements Serializable {
    @AllArgsConstructor
    @FieldDefaults(level = AccessLevel.PRIVATE)
    @Schema(description = "Object containing information of a SemanticNode and also structuring the layout with children.")
-    public static class EntryData implements Serializable {
+    public static class EntryData {

        @Schema(description = "Type of the semantic node.", allowableValues = {"DOCUMENT", "SECTION", "PARAGRAPH", "HEADLINE", "TABLE", "TABLE_CELL", "HEADER", "FOOTER", "IMAGE"})
        NodeType type;
@@ -1,7 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

-import java.io.Serializable;
-
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,7 +14,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@Schema(description = "Object containing text information of a specific text block. A document is split into multiple text blocks, which are supposed to be read in order. Every text block can only occur on a single page.")
-public class DocumentTextData implements Serializable {
+public class DocumentTextData {

    @Schema(description = "Identifier of the text block.")
    Long id;
@@ -1,9 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

-import java.io.Serializable;
 import java.util.Locale;

-public enum NodeType implements Serializable {
+public enum NodeType {
    DOCUMENT,
    SECTION,
    HEADLINE,
@@ -20,6 +20,9 @@ public record LayoutParsingRequest(
        @NonNull String originFileStorageId,//
        @Schema(description = "Optional Path to the table extraction file.")//
        Optional<String> tablesFileStorageId,//
+
+        @Schema(description= "Optional Path to the the table parsing service file")
+        Optional<String> tableExtractorFileId,
        @Schema(description = "Optional Path to the image classification file.")//
        Optional<String> imagesFileStorageId,//

@@ -1,6 +1,6 @@
 plugins {
    id("com.knecon.fforesight.java-conventions")
-    id("io.freefair.lombok") version "8.4"
+    id("io.freefair.lombok") version "8.2.2"
 }

 description = "layoutparser-service-processor"
@@ -10,14 +10,13 @@ val pdfBoxVersion = "3.0.0"

 dependencies {
    implementation(project(":layoutparser-service-internal-api"))
-    implementation(project(":viewer-doc-processor"))

    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0") {
        exclude("org.springframework.boot", "spring-boot-starter-security")
        exclude("org.springframework.boot", "spring-boot-starter-validation")
    }
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
+    implementation("com.iqser.red.commons:storage-commons:2.40.0")

    implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
@@ -3,17 +3,13 @@ package com.knecon.fforesight.service.layoutparser.processor;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
-import java.io.File;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;

-import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
@@ -27,7 +23,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
@@ -35,8 +30,11 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageB
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.TableExtractorResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
@@ -49,136 +47,109 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
-import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;

-import io.micrometer.observation.Observation;
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.annotation.Observed;
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

-@SuppressWarnings("PMD.CloseResource")
@Slf4j
@Service
@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class LayoutParsingPipeline {

-    ImageServiceResponseAdapter imageServiceResponseAdapter;
-    CvTableParsingAdapter cvTableParsingAdapter;
-    LayoutParsingStorageService layoutParsingStorageService;
-    SectionsBuilderService sectionsBuilderService;
-    TaasClassificationService taasClassificationService;
-    RedactManagerClassificationService redactManagerClassificationService;
-    DocuMineClassificationService docuMineClassificationService;
-    SimplifiedSectionTextService simplifiedSectionTextService;
-    BodyTextFrameService bodyTextFrameService;
-    RulingCleaningService rulingCleaningService;
-    TableExtractionService tableExtractionService;
-    TaasBlockificationService taasBlockificationService;
-    DocuMineBlockificationService docuMineBlockificationService;
-    RedactManagerBlockificationService redactManagerBlockificationService;
-    LayoutGridService layoutGridService;
-    ObservationRegistry observationRegistry;
-    DocstrumSegmentationService docstrumSegmentationService;
+    private final ImageServiceResponseAdapter imageServiceResponseAdapter;
+    private final CvTableParsingAdapter cvTableParsingAdapter;
+    private final LayoutParsingStorageService layoutParsingStorageService;
+    private final TableExtractorResponseAdapter tableExtractorResponseAdapter;
+    private final SectionsBuilderService sectionsBuilderService;
+    private final TaasClassificationService taasClassificationService;
+    private final RedactManagerClassificationService redactManagerClassificationService;
+    private final DocuMineClassificationService docuMineClassificationService;
+    private final SimplifiedSectionTextService simplifiedSectionTextService;
+    private final BodyTextFrameService bodyTextFrameService;
+    private final RulingCleaningService rulingCleaningService;
+    private final TableExtractionService tableExtractionService;
+    private final TaasBlockificationService taasBlockificationService;
+    private final DocuMineBlockificationService docuMineBlockificationService;
+    private final RedactManagerBlockificationService redactManagerBlockificationService;
+    private final ViewerDocumentService viewerDocumentService;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {

        long start = System.currentTimeMillis();
-        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

-        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);
+        try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
+            ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
+            if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
+                imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            }

-        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+            TableExtractorResponse tableExtractorResponse = new TableExtractorResponse();
+            if (layoutParsingRequest.tableExtractorFileId().isPresent()) {
+                tableExtractorResponse = layoutParsingStorageService.getExtractedTableFile(layoutParsingRequest.tableExtractorFileId().get());
+            }
+
+            TableServiceResponse tableServiceResponse = new TableServiceResponse();
+            if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
+                tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+            }
+
+            ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
+                    originDocument,
+                    imageServiceResponse,
+                    tableServiceResponse,
+                    tableExtractorResponse);
+            Document documentGraph = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+
+            int numberOfPages = originDocument.getNumberOfPages();
+
+            layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
+            layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
+
+            Map<Integer, List<TableExtractorCells>> extractedTableCells = tableExtractorResponseAdapter.buildExtractedTablesPerPage(tableExtractorResponse);
+            try (var out = new ByteArrayOutputStream()) {
+                viewerDocumentService.createViewerDocument(originDocument, documentGraph, out,extractedTableCells ,false);
+                layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);
+            }
+
+            if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+                var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
+                layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
+            }
+
+            return LayoutParsingFinishedEvent.builder()
+                    .identifier(layoutParsingRequest.identifier())
+                    .numberOfPages(numberOfPages)
+                    .duration(System.currentTimeMillis() - start)
+                    .message(format("""
+                                    Layout parsing has finished in %.02f s.
+                                    identifiers: %s
+                                    %s
+                                    Files have been saved with Ids:
+                                    Structure: %s
+                                    Text: %s
+                                    Positions: %s
+                                    PageData: %s
+                                    Simplified Text: %s
+                                    Viewer Doc: %s""",
+                            ((float) (System.currentTimeMillis() - start)) / 1000,
+                            layoutParsingRequest.identifier(),
+                            buildSemanticNodeCountMessage(numberOfPages, documentGraph.buildSemanticNodeCounts()),
+                            layoutParsingRequest.structureFileStorageId(),
+                            layoutParsingRequest.textBlockFileStorageId(),
+                            layoutParsingRequest.positionBlockFileStorageId(),
+                            layoutParsingRequest.pageFileStorageId(),
+                            layoutParsingRequest.simplifiedTextStorageId(),
+                            layoutParsingRequest.viewerDocumentStorageId()))
+                    .build();
        }
-
-        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
-        }
-
-        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
-                originFile,
-                imageServiceResponse,
-                tableServiceResponse,
-                layoutParsingRequest.identifier().toString());
-
-        log.info("Building document graph for {}", layoutParsingRequest.identifier());
-
-        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
-
-        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
-
-        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false);
-
-        log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
-
-        layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
-        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
-
-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
-            log.info("Building research document data for {}", layoutParsingRequest.identifier());
-            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
-            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
-        }
-
-        if (!viewerDocumentFile.equals(originFile)) {
-            viewerDocumentFile.delete();
-        }
-        originFile.delete();
-
-        return LayoutParsingFinishedEvent.builder()
-                .identifier(layoutParsingRequest.identifier())
-                .numberOfPages(documentGraph.getNumberOfPages())
-                .duration(System.currentTimeMillis() - start)
-                .message(format("""
-                                Layout parsing has finished in %.02f s.
-                                identifiers: %s
-                                %s
-                                Files have been saved with Ids:
-                                Structure: %s
-                                Text: %s
-                                Positions: %s
-                                PageData: %s
-                                Simplified Text: %s
-                                Viewer Doc: %s""",
-                        ((float) (System.currentTimeMillis() - start)) / 1000,
-                        layoutParsingRequest.identifier(),
-                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId(),
-                        layoutParsingRequest.simplifiedTextStorageId(),
-                        layoutParsingRequest.viewerDocumentStorageId()))
-                .build();
-
-    }
-
-
-    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
-
-        AtomicReference<Document> documentReference = new AtomicReference<>();
-
-        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
-            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
-        });
-
-        return documentReference.get();
    }


@@ -197,36 +168,22 @@ public class LayoutParsingPipeline {


    @SneakyThrows
-    @Observed(name = "LayoutParsingPipeline", contextualName = "parse-layout")
    public ClassificationDocument parseLayout(LayoutParsingType layoutParsingType,
-                                              File originFile,
+                                              PDDocument originDocument,
                                              ImageServiceResponse imageServiceResponse,
                                              TableServiceResponse tableServiceResponse,
-                                              String identifier) {
+                                              TableExtractorResponse tableExtractorResponse) {

-        PDDocument originDocument = openDocument(originFile);
-        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
-
+        Map<Integer, List<TableExtractorCells>> extractedTableCells = tableExtractorResponseAdapter.buildExtractedTablesPerPage(tableExtractorResponse);
        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

+        originDocument.setAllSecurityToBeRemoved(true);
        long pageCount = originDocument.getNumberOfPages();

        for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-
-            if (pageNumber % 100 == 0) {
-                // re-open document every once in a while to save on RAM. This has no significant performance impact.
-                // This is due to PDFBox caching all images and some other stuff with Soft References. This dereferences them and forces the freeing of memory.
-                originDocument.close();
-                originDocument = openDocument(originFile);
-            }
-
-            if (pageNumber % 100 == 0 || pageNumber == pageCount || pageNumber == 1) {
-                log.info("Extracting text on Page {} for {}", pageNumber, identifier);
-            }
-
            classificationDocument.setPages(classificationPages);
            PDFLinesTextStripper stripper = new PDFLinesTextStripper();
            PDPage pdPage = originDocument.getPage(pageNumber - 1);
@@ -247,37 +204,11 @@ public class LayoutParsingPipeline {
            PDRectangle cropbox = pdPage.getCropBox();
            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());

-            // Docstrum
-            AtomicInteger num = new AtomicInteger(pageNumber);
-            var zones = docstrumSegmentationService.segmentPage(stripper.getTextPositionSequences());
-
-            List<AbstractPageBlock> pageBlocks = new ArrayList<>();
-            AtomicInteger numOnPage = new AtomicInteger(1);
-//            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-            zones.forEach(zone -> {
-
-                List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-                zone.getLines().forEach(line -> {
-                    line.getWords().forEach(word -> {
-                        textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), num.get()));
-                    });
-                });
-
-                var cps = redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-                cps.getTextBlocks().forEach(cp -> {
-                    pageBlocks.add(redactManagerBlockificationService.buildTextBlock(((TextPageBlock) cp).getSequences(), numOnPage.getAndIncrement()));
-                });
-            });
-
-//            ClassificationPage classificationPage = switch (layoutParsingType) {
-//                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//                case TAAS -> taasBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//                case DOCUMINE -> docuMineBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//            };
-
-            ClassificationPage classificationPage = new ClassificationPage(pageBlocks);
-
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            };
            classificationPage.setCleanRulings(cleanRulings);
            classificationPage.setRotation(rotation);
            classificationPage.setLandscape(isLandscape);
@@ -302,52 +233,21 @@ public class LayoutParsingPipeline {
            classificationPages.add(classificationPage);
        }

-        originDocument.close();
-
-        log.info("Calculating BodyTextFrame for {}", identifier);
        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
-        log.info("Classify TextBlocks for {}", identifier);
+
        switch (layoutParsingType) {
            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }

-        List<ClassificationSection> sections = new ArrayList<>();
-        for (var page : classificationPages) {
-            page.getTextBlocks().forEach(block -> {
-                block.setPage(page.getPageNumber());
-                var section = sectionsBuilderService.buildTextBlock(List.of(block), "a");
-                sections.add(section);
-            });
-        }
-        classificationDocument.setSections(sections);
+        sectionsBuilderService.buildSections(classificationDocument);
+        sectionsBuilderService.addImagesToSections(classificationDocument);

-        log.info("Building Sections for {}", identifier);
-//        sectionsBuilderService.buildSections(classificationDocument);
-//        sectionsBuilderService.addImagesToSections(classificationDocument);
        return classificationDocument;
    }


-    private void addNumberOfPagesToTrace(int numberOfPages, long size) {
-
-        if (observationRegistry.getCurrentObservation() != null) {
-            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
-            observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileSize", String.valueOf(size));
-        }
-    }
-
-
-    @SneakyThrows
-    private PDDocument openDocument(File originFile) {
-
-        PDDocument document = Loader.loadPDF(originFile);
-        document.setAllSecurityToBeRemoved(true);
-        return document;
-    }
-
-
    private Map<String, List<Rectangle2D>> convertMarkedContents(List<PDMarkedContent> pdMarkedContents) {

        Map<String, List<Rectangle2D>> markedContentBboxes = new HashMap<>();
@@ -1,24 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor;

-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;

-import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
-
-import io.micrometer.observation.ObservationRegistry;
-
@Configuration
@ComponentScan
 public class LayoutParsingServiceProcessorConfiguration {

-
-    @Bean
-    @Autowired
-    public ViewerDocumentService viewerDocumentService(ObservationRegistry registry) {
-
-        return new ViewerDocumentService(registry);
-    }
-
 }
@@ -1,15 +1,17 @@
 package com.knecon.fforesight.service.layoutparser.processor;

+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
-import java.util.Optional;

+import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;
@@ -21,10 +23,10 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Si
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

-import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -37,24 +39,17 @@ public class LayoutParsingStorageService {
    private final StorageService storageService;
    private final ObjectMapper objectMapper;

-    @Observed(name = "LayoutParsingStorageService", contextualName = "get-origin-file")
-    public File getOriginFile(String storageId) throws IOException {

-        File tempFile = createTempFile("document", ".pdf");
-        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
-        return tempFile;
-    }
+    public PDDocument getOriginFile(String storageId) throws IOException {

-
-    @Observed(name = "LayoutParsingStorageService", contextualName = "get-viewer-doc-file")
-    public Optional<File> getViewerDocFile(String storageId) throws IOException {
-
-        if (!storageService.objectExists(TenantContext.getTenantId(), storageId)) {
-            return Optional.empty();
+        try (var originDocumentInputStream = getObject(storageId)) {
+            File tempFile = createTempFile("document", ".pdf");
+            try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
+                IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
+            }
+            return Loader.loadPDF(tempFile);
        }
-        File tempFile = createTempFile("viewerDocument", ".pdf");
-        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
-        return Optional.of(tempFile);
    }


@@ -69,6 +64,16 @@ public class LayoutParsingStorageService {
    }


+    public TableExtractorResponse getExtractedTableFile(String storageId) throws IOException {
+
+        try (InputStream inputStream = getObject(storageId)) {
+            TableExtractorResponse tableExtractorResponse = objectMapper.readValue(inputStream, TableExtractorResponse.class);
+            inputStream.close();
+            return tableExtractorResponse;
+        }
+    }
+
+
    public TableServiceResponse getTablesFile(String storageId) throws IOException {

        try (var tableClassificationStream = getObject(storageId)) {
@@ -80,7 +85,6 @@ public class LayoutParsingStorageService {
    }


-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), documentData.getDocumentStructure());
@@ -121,7 +125,6 @@ public class LayoutParsingStorageService {
    }


-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-simplified-text")
    public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {

        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
@@ -139,10 +142,10 @@ public class LayoutParsingStorageService {


    @SneakyThrows
-    @Observed(name = "LayoutParsingStorageService", contextualName = "store-viewer-document")
-    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, File out) {
+    public void storeViewerDocument(LayoutParsingRequest layoutParsingRequest, ByteArrayOutputStream out) {
+
+        try (var in = new ByteArrayInputStream(out.toByteArray())) {

-        try (var in = new FileInputStream(out)) {
            storageService.storeObject(TenantContext.getTenantId(), layoutParsingRequest.viewerDocumentStorageId(), in);
        }
    }
@@ -12,7 +12,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRul
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;

 import lombok.Data;
-import lombok.Getter;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;

@@ -21,7 +20,6 @@ import lombok.RequiredArgsConstructor;
 public class ClassificationPage {

    @NonNull
-    @Getter
    private List<AbstractPageBlock> textBlocks;

    private List<ClassifiedImage> images = new ArrayList<>();
@@ -19,4 +19,5 @@ public class PageContents {
    Rectangle2D cropBox;
    Rectangle2D mediaBox;
    List<Ruling> rulings;
+
 }
@@ -12,7 +12,6 @@ import lombok.Setter;

@Setter
@EqualsAndHashCode
-@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
 public class Boundary implements Comparable<Boundary> {

    private int start;
@@ -109,11 +108,13 @@ public class Boundary implements Comparable<Boundary> {
        return splitBoundaries;
    }

+
    public IntStream intStream() {

        return IntStream.range(start, end);
    }

+
    public static Boundary merge(Collection<Boundary> boundaries) {

        int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
@@ -10,6 +10,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.amazonaws.services.kms.model.NotFoundException;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -83,7 +84,7 @@ public class Document implements GenericSemanticNode {
    @Override
    public Headline getHeadline() {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build());
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseThrow(() -> new NotFoundException("No Headlines found in this document!"));
    }


@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Footer implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -67,14 +62,4 @@ public class Footer implements GenericSemanticNode {
        return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Header implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public boolean isLeaf() {
@@ -67,14 +62,4 @@ public class Header implements GenericSemanticNode {
        return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -36,9 +34,6 @@ public class Headline implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -74,14 +69,4 @@ public class Headline implements GenericSemanticNode {
        return this;
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -34,9 +32,6 @@ public class Paragraph implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -65,14 +60,4 @@ public class Paragraph implements GenericSemanticNode {
        return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,9 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
@@ -37,9 +35,6 @@ public class Section implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -79,14 +74,4 @@ public class Section implements GenericSemanticNode {
                .orElseGet(() -> getParent().getHeadline());
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -398,11 +398,12 @@ public interface SemanticNode {
     */
    default Map<Page, Rectangle2D> getBBox() {

+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        if (isLeaf()) {
-            return getBBoxFromLeafTextBlock();
+            return getBBoxFromLeafTextBlock(bBoxPerPage);
        }

-        return getBBoxFromChildren();
+        return getBBoxFromChildren(bBoxPerPage);
    }


@@ -425,31 +426,25 @@ public interface SemanticNode {

    /**
     * TODO: this produces unwanted results for sections spanning multiple columns.
-     * Computes the Union of the bounding boxes of all children recursively.
+     *
+     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of the BoundingBoxes of all children
     */
-    private Map<Page, Rectangle2D> getBBoxFromChildren() {
+    private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
-        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
-        for (Page page : pages) {
-            Rectangle2D bBoxOnPage = childrenBBoxes.stream()
-                    .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
-                    .map(childBboxPerPage -> childBboxPerPage.get(page))
-                    .collect(RectangleTransformations.collectBBox());
-            bBoxPerPage.put(page, bBoxOnPage);
-        }
-        return bBoxPerPage;
+        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
+            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
+            return map2;
+        }).orElse(bBoxPerPage);
    }


    /**
+     * @param bBoxPerPage initial empty BoundingBox
     * @return The union of all BoundingBoxes of the TextBlock of this node
     */
-    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
+    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {

-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
        return bBoxPerPage;
@@ -2,12 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

 import static java.lang.String.format;

-import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -42,8 +40,6 @@ public class Table implements SemanticNode {
    @EqualsAndHashCode.Exclude
    Set<RedactionEntity> entities = new HashSet<>();

-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;

    /**
     * Streams all entities in this table, that appear in a row, which contains any of the provided strings.
@@ -211,6 +207,7 @@ public class Table implements SemanticNode {
        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
    }

+
    /**
     * Streams all TableCells row-wise and filters them with header == true.
     *
@@ -315,12 +312,5 @@ public class Table implements SemanticNode {

        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
    }
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {

-        if (bBoxCache == null) {
-            bBoxCache = SemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
 }
@@ -109,10 +109,7 @@ public class AtomicTextBlock implements TextBlock {
    }


-    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData,
-                                                          DocumentPositionData documentPositionData,
-                                                          SemanticNode parent,
-                                                          Page page) {
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData documentTextData, DocumentPositionData documentPositionData, SemanticNode parent, Page page) {

        return AtomicTextBlock.builder()
                .id(documentTextData.getId())
@@ -1,14 +1,12 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -50,6 +48,7 @@ public class TablePageBlock extends AbstractPageBlock {
        return getColCount() == 0 || getRowCount() == 0;
    }

+
    public List<List<Cell>> getRows() {

        if (rows == null) {
@@ -276,21 +275,17 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-
    public boolean intersects(Cell cell1, Cell cell2) {
+
        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
            return false;
        }
        double x0 = cell1.getX() + 2;
        double y0 = cell1.getY() + 2;
-        return (cell2.x + cell2.width > x0 &&
-                cell2.y + cell2.height > y0 &&
-                cell2.x < x0 + cell1.getWidth() -2 &&
-                cell2.y < y0 + cell1.getHeight() -2);
+        return (cell2.x + cell2.width > x0 && cell2.y + cell2.height > y0 && cell2.x < x0 + cell1.getWidth() - 2 && cell2.y < y0 + cell1.getHeight() - 2);
    }


-
    @Override
    public String getText() {

@@ -328,8 +323,6 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-
-
    public String getTextAsHtml() {

        StringBuilder sb = new StringBuilder();
@@ -45,9 +45,6 @@ public class RedTextPosition {
    @JsonIgnore
    private String fontName;

-    @JsonIgnore
-    private RedTextPosition parent;
-

    @SneakyThrows
    public static RedTextPosition fromTextPosition(TextPosition textPosition) {
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;

 import java.util.ArrayList;
 import java.util.List;
+
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;

 import lombok.Getter;
@@ -17,7 +17,6 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.Getter;
 import lombok.NoArgsConstructor;

@EqualsAndHashCode(callSuper = true)
@@ -28,7 +27,6 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {

    @Builder.Default
-    @Getter
    private List<TextPositionSequence> sequences = new ArrayList<>();

    @JsonIgnore
@@ -9,6 +9,8 @@ import java.util.stream.Collectors;

 import org.apache.pdfbox.text.TextPosition;

+import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;

@@ -55,18 +57,6 @@ public class TextPositionSequence implements CharSequence {
    }


-    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
-
-        this.textPositions = textPositions;
-        this.page = page;
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        this.isParagraphStart = false;
-    }
-
-
    @Override
    public int length() {

@@ -152,7 +142,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minX value
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMinXDirAdj() {

        return textPositions.get(0).getXDirAdj();
@@ -166,7 +157,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxX value
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMaxXDirAdj() {

        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
@@ -180,7 +172,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMinYDirAdj() {

        return textPositions.get(0).getYDirAdj() - getTextHeight();
@@ -194,7 +187,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getMaxYDirAdj() {

        return textPositions.get(0).getYDirAdj();
@@ -202,24 +196,32 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getTextHeight() {

        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getHeight() {

        return getMaxYDirAdj() - getMinYDirAdj();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getWidth() {

        return getMaxXDirAdj() - getMinXDirAdj();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getFont() {

        if (textPositions.get(0).getFontName() == null) {
@@ -229,6 +231,8 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public String getFontStyle() {

        if (textPositions.get(0).getFontName() == null) {
@@ -248,12 +252,16 @@ public class TextPositionSequence implements CharSequence {
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getFontSize() {

        return textPositions.get(0).getFontSizeInPt();
    }


+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    public float getSpaceWidth() {

        return textPositions.get(0).getWidthOfSpace();
@@ -269,7 +277,8 @@ public class TextPositionSequence implements CharSequence {
     *
     * @return bounding box of the word in Pdf Coordinate System
     */
-
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
    @SneakyThrows
    public Rectangle getRectangle() {

@@ -0,0 +1,8 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.Color;
+import java.awt.geom.Line2D;
+
+public record ColoredLine(Line2D line, Color color) {
+
+}
@@ -1,8 +1,8 @@
-package com.knecon.fforesight.service.viewerdoc.model;
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;

 import java.awt.Color;
 import java.awt.geom.Rectangle2D;

-public record ColoredRectangle(Rectangle2D rectangle2D, Color color, float lineWidth) {
+public record ColoredRectangle(Rectangle2D rectangle2D, Color color) {

 }
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.viewerdoc.model;
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;

 import java.awt.Color;
 import java.awt.geom.Rectangle2D;
@@ -1,7 +1,7 @@
-package com.knecon.fforesight.service.viewerdoc.model;
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;

-import java.util.HashMap;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.List;

 import lombok.AccessLevel;
 import lombok.Getter;
@@ -12,15 +12,15 @@ import lombok.experimental.FieldDefaults;
 public class LayoutGrid {

    int numberOfPages;
-    Map<Integer, VisualizationsOnPage> visualizationsPerPages;
+    List<VisualizationsOnPage> visualizationsPerPages;


    public LayoutGrid(int numberOfPages) {

        this.numberOfPages = numberOfPages;
-        this.visualizationsPerPages = new HashMap<>();
+        this.visualizationsPerPages = new ArrayList<>(numberOfPages);
        for (int i = 0; i < numberOfPages; i++) {
-            this.visualizationsPerPages.put(i, VisualizationsOnPage.builder().build());
+            this.visualizationsPerPages.add(VisualizationsOnPage.builder().pageNumber(i).build());
        }
    }

@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;
+
+import java.awt.geom.Point2D;
+
+public record PlacedText(String text, Point2D lineStart) {
+
+}
@@ -1,4 +1,4 @@
-package com.knecon.fforesight.service.viewerdoc.model;
+package com.knecon.fforesight.service.layoutparser.processor.model.visualization;

 import java.util.LinkedList;
 import java.util.List;
@@ -13,7 +13,7 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class VisualizationsOnPage {

-    boolean makePathsInvisible;
+    int pageNumber;
    @Builder.Default
    List<PlacedText> placedTexts = new LinkedList<>();
    @Builder.Default
@@ -9,10 +9,10 @@ import java.util.Map;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;

 import lombok.RequiredArgsConstructor;

@@ -20,8 +20,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
 public class ImageServiceResponseAdapter {

-
-    public Map<Integer, List<ClassifiedImage>> buildClassifiedImagesPerPage(ImageServiceResponse imageServiceResponse ) {
+    public Map<Integer, List<ClassifiedImage>> buildClassifiedImagesPerPage(ImageServiceResponse imageServiceResponse) {

        Map<Integer, List<ClassifiedImage>> images = new HashMap<>();
        imageServiceResponse.getData().forEach(imageMetadata -> {
@@ -0,0 +1,74 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTable;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.ExtractedTableData;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableData;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class TableExtractorResponseAdapter {
+
+    public Map<Integer, List<TableExtractorCells>> buildExtractedTablesPerPage(TableExtractorResponse tableExtractorResponse) {
+        Map<Integer, List<TableExtractorCells>> tableCells = new HashMap<>();
+        tableExtractorResponse.getData()
+                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_number(), tableCell -> new ArrayList<>())
+                        .addAll(convertTableCells(tableData.getTables())));
+
+        return tableCells;
+
+    }
+
+    public List<TableExtractorCells> convertTableCells(List<ExtractedTable> tableObjects) {
+
+        List<TableExtractorCells> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            TableExtractorCells tableCells = new TableExtractorCells();
+            tableCells.setX0(t.getTable().getBbox().get(0));
+            tableCells.setX1(t.getTable().getBbox().get(2));
+            tableCells.setY0(t.getTable().getBbox().get(1));
+            tableCells.setY1(t.getTable().getBbox().get(3));
+            tableCells.setWidth(tableCells.getX1()- tableCells.getX0());
+            tableCells.setHeight(tableCells.getY1()- tableCells.getY0());
+            tableCells.setLabel(t.getTable().getLabel());
+            log.info("Parsed table cell {} with label {}",tableCells, tableCells.getLabel());
+            parsedTableCells.add(tableCells);
+            t.getObjects().forEach(o -> {
+                TableExtractorCells objectCell = new TableExtractorCells();
+                objectCell.setX0(t.getTable().getBbox().get(0));
+                objectCell.setX1(t.getTable().getBbox().get(2));
+                objectCell.setY0(t.getTable().getBbox().get(1));
+                objectCell.setY1(t.getTable().getBbox().get(3));
+                objectCell.setWidth(objectCell.getX1()- objectCell.getX0());
+                objectCell.setHeight(objectCell.getY1()- objectCell.getY0());
+                objectCell.setLabel(o.getLabel());
+                log.info("Parsed object cell {} with label {}",objectCell, objectCell.getLabel());
+                parsedTableCells.add(objectCell);
+            });
+        });
+
+        return parsedTableCells;
+
+    }
+
+
+}
@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExtractedTable {
+
+    private boolean rotated;
+    private ExtractedTableData table;
+    private List<ExtractedTableData> objects;
+
+}
@@ -0,0 +1,20 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExtractedTableData {
+
+    private String label;
+    private float score;
+    private List<Float> bbox;
+
+}
@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorCells {
+    private float x0;
+    private float y0;
+    private float x1;
+    private float y1;
+    private float width;
+    private float height;
+    private String label;
+
+}
@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorData {
+
+    private int page_number;
+    private int image;
+    private List<ExtractedTable> tables;
+}
@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TableExtractorResponse {
+
+    private String dossierId;
+    private String fileId;
+    private String targetFileExtension;
+    private String responseFileExtension;
+    private String X_TENANT_ID;
+    private List<TableExtractorData> data;
+
+}
@@ -25,6 +25,7 @@ public class BodyTextFrameService {
    private static final float RULING_HEIGHT_THRESHOLD = 0.15f; // multiplied with page height. Header/Footer Rulings must be within that border of the page.
    private static final float RULING_WIDTH_THRESHOLD = 0.75f; // multiplied with page width. Header/Footer Rulings must be at least that wide.

+
    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {

        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
@@ -155,8 +156,9 @@ public class BodyTextFrameService {
                        continue;
                    }

-                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || MarkedContentUtils.intersects(textBlock,
+                            page.getMarkedContentBboxPerType(),
+                            MarkedContentUtils.FOOTER)) {
                        continue;
                    }

@@ -22,7 +22,6 @@ public class DividingColumnDetectionService {

    public List<Rectangle2D> detectColumns(PageContents pageContents) {

-
        if (pageContents.getSortedTextPositionSequences().size() < 2) {
            return List.of(pageContents.getCropBox());
        }
@@ -72,11 +72,13 @@ public class GapDetectionService {
        return mirrorY(RectangleTransformations.toRectangle2D(textPosition.getRectangle()));
    }

+
    private static Rectangle2D mirrorY(Rectangle2D rectangle2D) {

        return new Rectangle2D.Double(rectangle2D.getX(), Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY()), rectangle2D.getWidth(), Math.abs(rectangle2D.getHeight()));
    }

+
    private static void addGapToLine(Rectangle2D currentTextPosition, Rectangle2D previousTextPosition, XGapsContext context) {

        context.gapsInCurrentLine.add(new Rectangle2D.Double(previousTextPosition.getMaxX(),
@@ -6,7 +6,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
 import java.util.stream.Stream;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;

@@ -51,7 +50,9 @@ public class GapsAcrossLinesService {
        }

        return columnFactory.outputGaps.stream()
-                .filter(gapAcrossLines -> columnFactory.outputGaps.stream().filter(gapAcrossLines::intersectsX).noneMatch(gapAcrossLines1 -> gapAcrossLines1.lineCount > gapAcrossLines.lineCount))
+                .filter(gapAcrossLines -> columnFactory.outputGaps.stream()
+                        .filter(gapAcrossLines::intersectsX)
+                        .noneMatch(gapAcrossLines1 -> gapAcrossLines1.lineCount > gapAcrossLines.lineCount))
                .filter(gapAcrossLines -> Math.abs(gapAcrossLines.rectangle2D.getMinX() - mainBodyTextFrame.getMinX()) > DISTANCE_TO_BORDER_THRESHOLD)
                .filter(gapAcrossLines -> Math.abs(gapAcrossLines.rectangle2D.getMaxX() - mainBodyTextFrame.getMaxX()) > DISTANCE_TO_BORDER_THRESHOLD)
                .map(GapAcrossLines::getRectangle2D)
@@ -6,8 +6,8 @@ import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
 import com.knecon.fforesight.service.layoutparser.processor.model.LineInformation;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -16,8 +16,7 @@ public class MainBodyTextFrameExtractionService {

    public Rectangle2D calculateMainBodyTextFrame(LineInformation lineInformation) {

-        Rectangle2D mainBodyTextFrame = lineInformation.getLineBBox().stream()
-                .collect(RectangleTransformations.collectBBox());
+        Rectangle2D mainBodyTextFrame = lineInformation.getLineBBox().stream().collect(RectangleTransformations.collectBBox());

        return RectangleTransformations.pad(mainBodyTextFrame, mainBodyTextFrame.getWidth() * TEXT_FRAME_PAD_WIDTH, mainBodyTextFrame.getHeight() * TEXT_FRAME_PAD_HEIGHT);
    }
@@ -52,7 +52,7 @@ public class PageContentExtractor {
                        stripper.getRulings()));
            }
        }
-    
+
        return textPositionSequencesPerPage;
    }

@@ -240,7 +240,7 @@ public class SectionsBuilderService {
    }


-    public ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
+    private ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {

        ClassificationSection section = new ClassificationSection();

@@ -5,9 +5,9 @@ import java.util.List;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;

@Service
 public class SimplifiedSectionTextService {
@@ -23,4 +23,5 @@ public class SimplifiedSectionTextService {

        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
    }
+
 }
@@ -57,7 +57,7 @@ public class RedactManagerBlockificationService {
            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());

-            if (prev != null && (splitByDir || isSplitByRuling)) {
+            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {

                Orientation prevOrientation = null;
                if (!chunkBlockList.isEmpty()) {
@@ -167,7 +167,7 @@ public class RedactManagerBlockificationService {
    }


-    public TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    private TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {

        TextPageBlock textBlock = null;

@@ -1,9 +1,20 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;

-
 // TODO: figure out, why this fails the build
 // import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;

+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
@@ -11,12 +22,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;

@Service
@SuppressWarnings("all")
@@ -83,13 +88,13 @@ public class TaasBlockificationService {
                continue;
            }

-
            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
            boolean isListIdentifier = listIdentifierPattern.find();

            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;

-            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
+            boolean sameFont = previousTextBlock.getMostPopularWordFont()
+                    .equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
 //            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;

            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
@@ -119,8 +124,9 @@ public class TaasBlockificationService {
            }
            alreadyMerged.add(textPageBlock);
            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
-                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
-                    .toList());
+                    textPageBlocks.stream()
+                            .filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2))
+                            .peek(alreadyMerged::add)).toList());
        }
        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
    }
@@ -163,8 +169,7 @@ public class TaasBlockificationService {
        while (itty.hasNext()) {
            TextPageBlock block = (TextPageBlock) itty.next();

-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(
-                    block.getMaxY(),
+            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
                previous.add(block);
@@ -189,7 +194,6 @@ public class TaasBlockificationService {
        TextPositionSequence prev = null;
        // TODO: make static final constant

-
        boolean wasSplitted = false;
        Float splitX1 = null;
        for (TextPositionSequence word : textPositions) {
@@ -5,7 +5,6 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -13,6 +12,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -63,16 +63,16 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
-        ) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())
-        ) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -11,6 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RedactManagerClassificationService {

-
    public void classifyDocument(ClassificationDocument document) {

        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
@@ -52,14 +51,16 @@ public class RedactManagerClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.HEADER);

-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 import java.util.List;
 import java.util.regex.Pattern;

-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -12,6 +11,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;

 import lombok.RequiredArgsConstructor;
@@ -27,7 +27,6 @@ public class TaasClassificationService {

    public void classifyDocument(ClassificationDocument document) {

-
        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
@@ -57,11 +56,13 @@ public class TaasClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation())) {
            textBlock.setClassification(PageBlockType.HEADER);
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation())) {
            textBlock.setClassification(PageBlockType.FOOTER);
        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
@@ -1,153 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.LineBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.NearestNeighbourService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ReadingOrderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.SpacingService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ZoneBuilderService;
-
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class DocstrumSegmentationService {
-
-    private static final double MAX_VERTICAL_MERGE_DIST = 0.5;
-
-    private final NearestNeighbourService nearestNeighbourService;
-    private final SpacingService spacingService;
-    private final LineBuilderService lineBuilderService;
-    private final ZoneBuilderService zoneBuilderService;
-    private final ReadingOrderService readingOrderService;
-
-
-    public List<Zone> segmentPage(List<TextPositionSequence> textPositions) {
-
-        var positions = textPositions.stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
-
-        var characters = positions.stream().map(Character::new).collect(Collectors.toList());
-
-        nearestNeighbourService.findNearestNeighbors(characters);
-
-        var characterSpacing = spacingService.computeCharacterSpacing(characters);
-        var lineSpacing = spacingService.computeLineSpacing(characters);
-
-        var lines = lineBuilderService.buildLines(characters, characterSpacing, lineSpacing);
-
-        var zones = zoneBuilderService.buildZones(lines, characterSpacing, lineSpacing);
-
-        zones = mergeLinesInZones(zones, characterSpacing, Double.NEGATIVE_INFINITY, 0.0, 0.0, lineSpacing * MAX_VERTICAL_MERGE_DIST);
-
-        return readingOrderService.resolve(zones, false);
-    }
-
-//    private List<Zone> mergeZones(List<Zone> zones, double tolerance) {
-//
-//        List<BxBounds> bounds = new ArrayList<BxBounds>(zones.size());
-//        for (List<ComponentLine> zone : zones) {
-//            BxBoundsBuilder builder = new BxBoundsBuilder();
-//            for (ComponentLine line : zone) {
-//                for (Component component : line.getComponents()) {
-//                    builder.expand(component.getChunk().getBounds());
-//                }
-//            }
-//            bounds.add(builder.getBounds());
-//        }
-//
-//        List<List<ComponentLine>> outputZones = new ArrayList<List<ComponentLine>>();
-//        mainFor:
-//        for (int i = 0; i < zones.size(); i++) {
-//            for (int j = 0; j < zones.size(); j++) {
-//                if (i == j || bounds.get(j) == null || bounds.get(i) == null) {
-//                    continue;
-//                }
-//                if (BxModelUtils.contains(bounds.get(j), bounds.get(i), tolerance)) {
-//                    zones.get(j).addAll(zones.get(i));
-//                    bounds.set(i, null);
-//                    continue mainFor;
-//                }
-//            }
-//            outputZones.add(zones.get(i));
-//        }
-//        return outputZones;
-//    }
-
-
-    private List<Zone> mergeLinesInZones(List<Zone> zones,
-                                         double wordSpacing,
-                                         double minHorizontalDistance,
-                                         double maxHorizontalDistance,
-                                         double minVerticalDistance,
-                                         double maxVerticalDistance) {
-
-        List<Zone> outputZones = new ArrayList<>(zones.size());
-        for (Zone zone : zones) {
-            outputZones.add(mergeLinesInZone(zone, wordSpacing, minHorizontalDistance, maxHorizontalDistance, minVerticalDistance, maxVerticalDistance));
-        }
-        return outputZones;
-    }
-
-
-    private Zone mergeLinesInZone(Zone zone,
-                                  double wordSpacing,
-                                  double minHorizontalDistance,
-                                  double maxHorizontalDistance,
-                                  double minVerticalDistance,
-                                  double maxVerticalDistance) {
-
-        DisjointSets<Line> sets = new DisjointSets<>(zone.getLines());
-        for (int i = 0; i < zone.getLines().size(); i++) {
-            Line li = zone.getLines().get(i);
-            for (int j = i + 1; j < zone.getLines().size(); j++) {
-                Line lj = zone.getLines().get(j);
-                double hDist = li.horizontalDistance(lj);
-                double vDist = li.verticalDistance(lj);
-                if (minHorizontalDistance <= hDist && hDist <= maxHorizontalDistance && minVerticalDistance <= vDist && vDist <= maxVerticalDistance) {
-                    sets.union(li, lj);
-                } else if (minVerticalDistance <= vDist && vDist <= maxVerticalDistance && Math.abs(hDist - Math.min(li.getLength(), lj.getLength())) < 0.1) {
-                    boolean componentOverlap = false;
-                    int overlappingCount = 0;
-                    for (Character ci : li.getCharacters()) {
-                        for (Character cj : lj.getCharacters()) {
-                            double dist = ci.overlappingDistance(cj);
-                            if (dist > 2) {
-                                componentOverlap = true;
-                            }
-                            if (dist > 0) {
-                                overlappingCount++;
-                            }
-                        }
-                    }
-                    if (!componentOverlap && overlappingCount <= 2) {
-                        sets.union(li, lj);
-                    }
-                }
-            }
-        }
-        List<Line> outputZone = new ArrayList<>();
-        for (Set<Line> group : sets) {
-            List<Character> components = new ArrayList<>();
-            for (Line line : group) {
-                components.addAll(line.getCharacters());
-            }
-            components.sort(Comparator.comparingDouble(Character::getX));
-
-            outputZone.add(new Line(components, wordSpacing));
-        }
-        return new Zone(outputZone);
-    }
-
-}
@@ -1,32 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-public class AngleFilter {
-
-    protected double lowerAngle;
-    protected double upperAngle;
-
-
-    public AngleFilter(double lowerAngle, double upperAngle) {
-
-        if (lowerAngle < -Math.PI / 2) {
-            lowerAngle += Math.PI;
-        }
-        if (upperAngle >= Math.PI / 2) {
-            upperAngle -= Math.PI;
-        }
-
-        this.lowerAngle = lowerAngle;
-        this.upperAngle = upperAngle;
-    }
-
-
-    public boolean matches(Neighbor neighbor) {
-
-        if (lowerAngle <= upperAngle) {
-            return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
-        } else {
-            return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
-        }
-    }
-
-}
@@ -1,48 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.awt.geom.Rectangle2D;
-
-import lombok.Data;
-
-@Data
-public abstract class BoundingBox {
-
-    private Rectangle2D bBox;
-
-
-    public double getX() {
-
-        return bBox.getX();
-    }
-
-
-    public double getY() {
-
-        return bBox.getY();
-    }
-
-
-    public double getWidth() {
-
-        return bBox.getWidth();
-    }
-
-
-    public double getHeight() {
-
-        return bBox.getHeight();
-    }
-
-
-    public double getArea() {
-
-        return (bBox.getHeight() * bBox.getWidth());
-    }
-
-
-    public boolean contains(Rectangle2D contained, double tolerance) {
-
-        return bBox.getX() <= contained.getX() + tolerance && bBox.getY() <= contained.getY() + tolerance && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
-    }
-
-}
@@ -1,84 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-
-import lombok.Data;
-
-@Data
-public class Character {
-
-    private final double x;
-    private final double y;
-    private final RedTextPosition textPosition;
-
-    private List<Neighbor> neighbors = new ArrayList<>();
-
-
-    public Character(RedTextPosition chunk) {
-
-        this.x = chunk.getXDirAdj() + chunk.getWidthDirAdj() / 2;
-        this.y = chunk.getYDirAdj() + chunk.getHeightDir() / 2;
-        this.textPosition = chunk;
-    }
-
-
-    public double getHeight() {
-
-        return textPosition.getHeightDir();
-    }
-
-
-    public double distance(Character character) {
-
-        double dx = getX() - character.getX();
-        double dy = getY() - character.getY();
-        return Math.sqrt(dx * dx + dy * dy);
-    }
-
-
-    public double horizontalDistance(Character character) {
-
-        return Math.abs(getX() - character.getX());
-    }
-
-
-    public double verticalDistance(Character character) {
-
-        return Math.abs(getY() - character.getY());
-    }
-
-
-    public double overlappingDistance(Character other) {
-
-        double[] xs = new double[4];
-        double s = Math.sin(-0), c = Math.cos(-0);
-        xs[0] = c * x - s * y;
-        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
-        xs[2] = c * other.x - s * other.y;
-        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
-        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
-        Arrays.sort(xs);
-        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
-    }
-
-
-    public void setNeighbors(List<Neighbor> neighbors) {
-
-        this.neighbors = neighbors;
-    }
-
-
-    public double angle(Character character) {
-
-        if (getX() > character.getX()) {
-            return Math.atan2(getY() - character.getY(), getX() - character.getX());
-        } else {
-            return Math.atan2(character.getY() - getY(), character.getX() - getX());
-        }
-    }
-
-}
@@ -1,194 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.util.AbstractSet;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-public class DisjointSets<E> implements Iterable<Set<E>> {
-
-    private final Map<E, Entry<E>> map = new HashMap<>();
-
-
-    public DisjointSets(Collection<? extends E> collection) {
-
-        for (E element : collection) {
-            map.put(element, new Entry<E>(element));
-        }
-    }
-
-
-    public boolean areTogether(E e1, E e2) {
-
-        return map.get(e1).findRepresentative() == map.get(e2).findRepresentative();
-    }
-
-
-    public void union(E e1, E e2) {
-
-        Entry<E> r1 = map.get(e1).findRepresentative();
-        Entry<E> r2 = map.get(e2).findRepresentative();
-        if (r1 != r2) {
-            if (r1.size <= r2.size) {
-                r2.mergeWith(r1);
-            } else {
-                r1.mergeWith(r2);
-            }
-        }
-    }
-
-
-    @Override
-    public Iterator<Set<E>> iterator() {
-
-        return new Iterator<>() {
-
-            private final Iterator<Entry<E>> iterator = map.values().iterator();
-            private Entry<E> nextRepresentative;
-
-            {
-                findNextRepresentative();
-            }
-
-            @Override
-            public boolean hasNext() {
-
-                return nextRepresentative != null;
-            }
-
-
-            @Override
-            public Set<E> next() {
-
-                if (nextRepresentative == null) {
-                    throw new NoSuchElementException();
-                }
-                Set<E> result = nextRepresentative.asSet();
-                findNextRepresentative();
-                return result;
-            }
-
-
-            private void findNextRepresentative() {
-
-                while (iterator.hasNext()) {
-                    Entry<E> candidate = iterator.next();
-                    if (candidate.isRepresentative()) {
-                        nextRepresentative = candidate;
-                        return;
-                    }
-                }
-                nextRepresentative = null;
-            }
-
-
-            @Override
-            public void remove() {
-
-                throw new UnsupportedOperationException();
-            }
-
-        };
-    }
-
-
-    private static class Entry<E> {
-
-        private int size = 1;
-        private final E value;
-        private Entry<E> parent = this;
-        private Entry<E> next = null;
-        private Entry<E> last = this;
-
-
-        Entry(E value) {
-
-            this.value = value;
-        }
-
-
-        void mergeWith(Entry<E> otherRepresentative) {
-
-            size += otherRepresentative.size;
-            last.next = otherRepresentative;
-            last = otherRepresentative.last;
-            otherRepresentative.parent = this;
-        }
-
-
-        Entry<E> findRepresentative() {
-
-            Entry<E> representative = parent;
-            while (representative.parent != representative) {
-                representative = representative.parent;
-            }
-            for (Entry<E> entry = this; entry != representative; ) {
-                Entry<E> nextEntry = entry.parent;
-                entry.parent = representative;
-                entry = nextEntry;
-            }
-            return representative;
-        }
-
-
-        boolean isRepresentative() {
-
-            return parent == this;
-        }
-
-
-        Set<E> asSet() {
-
-            return new AbstractSet<E>() {
-
-                @Override
-                public Iterator<E> iterator() {
-
-                    return new Iterator<E>() {
-
-                        private Entry<E> nextEntry = findRepresentative();
-
-
-                        @Override
-                        public boolean hasNext() {
-
-                            return nextEntry != null;
-                        }
-
-
-                        @Override
-                        public E next() {
-
-                            if (nextEntry == null) {
-                                throw new NoSuchElementException();
-                            }
-                            E result = nextEntry.value;
-                            nextEntry = nextEntry.next;
-                            return result;
-                        }
-
-
-                        @Override
-                        public void remove() {
-
-                            throw new UnsupportedOperationException();
-                        }
-
-                    };
-                }
-
-
-                @Override
-                public int size() {
-
-                    return findRepresentative().size;
-                }
-            };
-        }
-
-    }
-
-}
@@ -1,91 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-public class Histogram {
-
-    private static final double EPSILON = 1.0e-6;
-    private final double min;
-    private final double resolution;
-    private double[] frequencies;
-
-
-    public Histogram(double minValue, double maxValue, double resolution) {
-
-        this.min = minValue - EPSILON;
-        double delta = maxValue - minValue + 2 * EPSILON;
-        int size = Math.max(1, (int) Math.round((maxValue - minValue) / resolution));
-        this.resolution = delta / size;
-        this.frequencies = new double[size];
-    }
-
-
-    public void kernelSmooth(double[] kernel) {
-
-        double[] newFrequencies = new double[frequencies.length];
-        int shift = (kernel.length - 1) / 2;
-        for (int i = 0; i < kernel.length; i++) {
-            int jStart = Math.max(0, i - shift);
-            int jEnd = Math.min(frequencies.length, frequencies.length + i - shift);
-            for (int j = jStart; j < jEnd; j++) {
-                newFrequencies[j - i + shift] += kernel[i] * frequencies[j];
-            }
-        }
-        frequencies = newFrequencies;
-    }
-
-
-    public double[] createGaussianKernel(double length, double stdDeviation) {
-
-        int r = (int) Math.round(length / resolution) / 2;
-        stdDeviation /= resolution;
-
-        int size = 2 * r + 1;
-        double[] kernel = new double[size];
-        double sum = 0;
-        double b = 2 * stdDeviation * stdDeviation;
-        double a = 1 / Math.sqrt(Math.PI * b);
-        for (int i = 0; i < size; i++) {
-            kernel[i] = a * Math.exp(-(i - r) * (i - r) / b);
-            sum += kernel[i];
-        }
-        for (int i = 0; i < size; i++) {
-            kernel[i] /= sum;
-        }
-        return kernel;
-    }
-
-
-    public void gaussianSmooth(double windowLength, double stdDeviation) {
-
-        kernelSmooth(createGaussianKernel(windowLength, stdDeviation));
-    }
-
-
-    public void add(double value) {
-
-        frequencies[(int) ((value - min) / resolution)] += 1.0;
-    }
-
-
-    public int getSize() {
-
-        return frequencies.length;
-    }
-
-
-    public double getPeakValue() {
-
-        int peakIndex = 0;
-        for (int i = 1; i < frequencies.length; i++) {
-            if (frequencies[i] > frequencies[peakIndex]) {
-                peakIndex = i;
-            }
-        }
-        int peakEndIndex = peakIndex + 1;
-        final double EPS = 0.0001;
-        while (peakEndIndex < frequencies.length && Math.abs(frequencies[peakEndIndex] - frequencies[peakIndex]) < EPS) {
-            peakEndIndex++;
-        }
-        return ((double) peakIndex + peakEndIndex) / 2 * resolution + min;
-    }
-
-}
@@ -1,165 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-
-import lombok.Data;
-
-@Data
-public class Line extends BoundingBox {
-
-    private static final double WORD_DISTANCE_MULTIPLIER = 0.2;
-
-    private final double x0;
-    private final double y0;
-
-    private final double x1;
-    private final double y1;
-
-    private final double height;
-
-    private final List<Character> characters;
-    private final List<TextPositionSequence> words = new ArrayList<>();
-
-
-    public Line(List<Character> characters, double wordSpacing) {
-
-        this.characters = characters;
-
-        if (characters.size() >= 2) {
-            // linear regression
-            double sx = 0.0, sxx = 0.0, sxy = 0.0, sy = 0.0;
-            for (Character character : characters) {
-                sx += character.getX();
-                sxx += character.getX() * character.getX();
-                sxy += character.getX() * character.getY();
-                sy += character.getY();
-            }
-            double b = (characters.size() * sxy - sx * sy) / (characters.size() * sxx - sx * sx);
-            double a = (sy - b * sx) / characters.size();
-
-            this.x0 = characters.get(0).getX();
-            this.y0 = a + b * this.x0;
-            this.x1 = characters.get(characters.size() - 1).getX();
-            this.y1 = a + b * this.x1;
-        } else {
-            Character character = characters.get(0);
-            double dx = character.getTextPosition().getWidthDirAdj() / 3;
-            double dy = dx * Math.tan(0);
-            this.x0 = character.getX() - dx;
-            this.x1 = character.getX() + dx;
-            this.y0 = character.getY() - dy;
-            this.y1 = character.getY() + dy;
-        }
-        height = computeHeight();
-        computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
-        buildBBox();
-    }
-
-
-    public double getAngle() {
-
-        return Math.atan2(y1 - y0, x1 - x0);
-    }
-
-
-    public double getLength() {
-
-        return Math.sqrt((x0 - x1) * (x0 - x1) + (y0 - y1) * (y0 - y1));
-    }
-
-
-    private double computeHeight() {
-
-        double sum = 0.0;
-        for (Character component : characters) {
-            sum += component.getHeight();
-        }
-        return sum / characters.size();
-    }
-
-
-    public double angularDifference(Line j) {
-
-        double diff = Math.abs(getAngle() - j.getAngle());
-        if (diff <= Math.PI / 2) {
-            return diff;
-        } else {
-            return Math.PI - diff;
-        }
-    }
-
-
-    public double horizontalDistance(Line other) {
-
-        double[] xs = new double[4];
-        xs[0] = x0;
-        xs[1] = x1;
-        xs[2] = other.x0;
-        xs[3] = other.x1;
-        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
-        Arrays.sort(xs);
-        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
-    }
-
-
-    public double verticalDistance(Line other) {
-
-        double ym = (y0 + y1) / 2;
-        double yn = (other.y0 + other.y1) / 2;
-        return Math.abs(ym - yn) / Math.sqrt(1);
-    }
-
-
-    private void computeWords(double wordSpacing) {
-
-        TextPositionSequence word = new TextPositionSequence();
-        Character previous = null;
-        for (Character current : characters) {
-            if (previous != null) {
-                double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
-                if (dist > wordSpacing) {
-                    words.add(word);
-                    word = new TextPositionSequence();
-                }
-            }
-            word.getTextPositions().add(current.getTextPosition());
-            previous = current;
-        }
-        words.add(word);
-    }
-
-
-    private void buildBBox() {
-
-        double minX = Double.POSITIVE_INFINITY;
-        double minY = Double.POSITIVE_INFINITY;
-        double maxX = Double.NEGATIVE_INFINITY;
-        double maxY = Double.NEGATIVE_INFINITY;
-
-        for (Character character : characters) {
-
-            minX = Math.min(minX, character.getTextPosition().getXDirAdj());
-            minY = Math.min(minY, character.getTextPosition().getYDirAdj());
-            maxX = Math.max(maxX, character.getTextPosition().getXDirAdj() + character.getTextPosition().getWidthDirAdj());
-            maxY = Math.max(maxY, character.getTextPosition().getYDirAdj() + character.getTextPosition().getHeightDir());
-
-        }
-
-        this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
-    }
-
-
-    public String toString() {
-
-        StringBuilder sb = new StringBuilder();
-        words.forEach(word -> sb.append(word.toString()).append(" "));
-        return sb.toString().trim();
-    }
-
-}
-
@@ -1,36 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import lombok.Getter;
-
-public class Neighbor {
-
-    @Getter
-    private final double distance;
-    @Getter
-    private final double angle;
-    private final Character originCharacter;
-    @Getter
-    private final Character character;
-
-
-    public Neighbor(Character neighbor, Character origin) {
-
-        this.distance = neighbor.distance(origin);
-        this.angle = neighbor.angle(origin);
-        this.character = neighbor;
-        this.originCharacter = origin;
-    }
-
-
-    public double getHorizontalDistance() {
-
-        return character.horizontalDistance(originCharacter);
-    }
-
-
-    public double getVerticalDistance() {
-
-        return character.verticalDistance(originCharacter);
-    }
-
-}
@@ -1,50 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Comparator;
-import java.util.List;
-
-import lombok.Data;
-
-@Data
-public class Zone extends BoundingBox {
-
-    private List<Line> lines;
-
-
-    public Zone(List<Line> lines) {
-
-        lines.sort(Comparator.comparingDouble(Line::getY));
-        this.lines = lines;
-        buildBBox();
-    }
-
-
-    public void buildBBox() {
-
-        double minX = Double.POSITIVE_INFINITY;
-        double minY = Double.POSITIVE_INFINITY;
-        double maxX = Double.NEGATIVE_INFINITY;
-        double maxY = Double.NEGATIVE_INFINITY;
-
-        for (Line line : lines) {
-
-            minX = Math.min(minX, line.getX());
-            minY = Math.min(minY, line.getY());
-            maxX = Math.max(maxX, line.getX() + line.getWidth());
-            maxY = Math.max(maxY, line.getY() + line.getHeight());
-
-        }
-
-        this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
-    }
-
-
-    public String toString() {
-
-        StringBuilder sb = new StringBuilder();
-        lines.forEach(line -> sb.append(line.toString()).append("\n"));
-        return sb.toString().trim();
-    }
-
-}
@@ -1,50 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-
-@Service
-public class LineBuilderService {
-
-    private static final double CHARACTER_SPACING_DISTANCE_MULTIPLIER = 3.5;
-    private static final double MAX_VERTICAL_CHARACTER_DISTANCE = 0.67;
-    private static final double ANGLE_TOLERANCE = Math.PI / 6;
-
-
-    public List<Line> buildLines(List<Character> characters, double characterSpacing, double lineSpacing) {
-
-        double maxHorizontalDistance = characterSpacing * CHARACTER_SPACING_DISTANCE_MULTIPLIER;
-        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_CHARACTER_DISTANCE;
-
-        DisjointSets<Character> sets = new DisjointSets<>(characters);
-        AngleFilter filter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
-
-        characters.forEach(character -> {
-            character.getNeighbors().forEach(neighbor -> {
-                double x = neighbor.getHorizontalDistance() / maxHorizontalDistance;
-                double y = neighbor.getVerticalDistance() / maxVerticalDistance;
-                if (filter.matches(neighbor) && Math.pow(x, 2) + Math.pow(y, 2) <= 1) {
-                    sets.union(character, neighbor.getCharacter());
-                }
-            });
-        });
-
-        List<Line> lines = new ArrayList<>();
-        sets.forEach(group -> {
-            List<Character> lineComponents = new ArrayList<>(group);
-            lineComponents.sort(Comparator.comparingDouble(Character::getX));
-            lines.add(new Line(lineComponents, characterSpacing));
-        });
-
-        return lines;
-    }
-
-}
@@ -1,78 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
-
-@Service
-public class NearestNeighbourService {
-
-    private static final int NUMBER_OF_NEIGHBOURS = 8;
-    private static final double STEP = 16.0;
-
-
-    public void findNearestNeighbors(List<Character> characters) {
-
-        if (characters.isEmpty()) {
-            return;
-        }
-
-        characters.sort(Comparator.comparingDouble(Character::getX));
-
-        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
-        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
-            maxNeighborCount = characters.size() - 1;
-        }
-
-        for (int i = 0; i < characters.size(); i++) {
-
-            List<Neighbor> candidates = new ArrayList<>();
-
-            int start = i;
-            int end = i + 1;
-
-            double distance = Double.POSITIVE_INFINITY;
-
-            for (double searchDistance = 0; searchDistance < distance; ) {
-
-                searchDistance += STEP;
-                boolean newCandidatesFound = false;
-
-                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
-                    start--;
-                    candidates.add(new Neighbor(characters.get(start), characters.get(i)));
-                    clearLeastDistant(candidates, maxNeighborCount);
-                    newCandidatesFound = true;
-                }
-
-                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
-                    candidates.add(new Neighbor(characters.get(end), characters.get(i)));
-                    clearLeastDistant(candidates, maxNeighborCount);
-                    end++;
-                    newCandidatesFound = true;
-                }
-
-                if (newCandidatesFound && candidates.size() >= maxNeighborCount) {
-                    distance = candidates.get(maxNeighborCount - 1).getDistance();
-                }
-            }
-            clearLeastDistant(candidates, maxNeighborCount);
-            characters.get(i).setNeighbors(new ArrayList<>(candidates));
-        }
-    }
-
-
-    private void clearLeastDistant(List<Neighbor> candidates, int maxNeighborCount) {
-
-        if (candidates.size() > maxNeighborCount) {
-            candidates.sort(Comparator.comparingDouble(Neighbor::getDistance));
-            candidates.remove(candidates.remove(candidates.size() - 1));
-        }
-    }
-
-}
@@ -1,99 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.ListIterator;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
-
-@Service
-public class ReadingOrderService {
-
-    private static final double THRESHOLD = 1;
-
-
-    public List<Zone> resolve(List<Zone> zones, boolean yxOrder) {
-
-        if (zones.isEmpty() || zones.size() == 1) {
-            return zones;
-        }
-
-        if (yxOrder) {
-            zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                    .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-            return zones;
-        }
-
-        return resolveMultiColumnReadingOder(zones);
-    }
-
-
-    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
-
-        // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
-        // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
-
-        double minX = Double.POSITIVE_INFINITY;
-        double maxX = Double.NEGATIVE_INFINITY;
-
-        for (Zone zone : zones) {
-            if (zone.getX() < minX) {
-                minX = zone.getX();
-            }
-            if (zone.getX() + zone.getWidth() > maxX) {
-                maxX = zone.getX() + zone.getWidth();
-            }
-        }
-
-        double midLineXCoordinate = (minX + maxX) / 2;
-
-        List<Zone> leftOf = new ArrayList<>();
-        List<Zone> rightOf = new ArrayList<>();
-        List<Zone> middle = new ArrayList<>();
-        for (Zone zone : zones) {
-            if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
-                leftOf.add(zone);
-            } else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
-                rightOf.add(zone);
-            } else {
-                middle.add(zone);
-            }
-        }
-
-        leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        List<Zone> sortedZones = new ArrayList<>();
-        sortedZones.addAll(leftOf);
-        sortedZones.addAll(rightOf);
-
-        ListIterator<Zone> itty = middle.listIterator();
-
-        while (itty.hasNext()) {
-            Zone current = itty.next();
-            for (int i = 0; i < sortedZones.size(); i++) {
-                if (current.getY() < sortedZones.get(i).getY()) {
-                    sortedZones.add(i, current);
-                    itty.remove();
-                    break;
-                }
-            }
-        }
-
-        sortedZones.addAll(middle);
-
-        return sortedZones;
-    }
-
-}
@@ -1,56 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Histogram;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
-
-@Service
-public class SpacingService {
-
-    private static final double SPACING_HISTOGRAM_RESOLUTION = 0.5;
-    private static final double SPACING_HISTOGRAM_SMOOTHING_LENGTH = 2.5;
-    private static final double SPACING_HIST_SMOOTHING_STANDARD_DEVIATION = 0.5;
-    private static final double ANGLE_TOLERANCE = Math.PI / 6;
-
-
-    public double computeCharacterSpacing(List<Character> components) {
-
-        return computeSpacing(components, 0);
-    }
-
-
-    public double computeLineSpacing(List<Character> components) {
-
-        return computeSpacing(components, Math.PI / 2);
-    }
-
-
-    private double computeSpacing(List<Character> components, double angle) {
-
-        double maxDistance = Double.NEGATIVE_INFINITY;
-
-        for (Character component : components) {
-            for (Neighbor neighbor : component.getNeighbors()) {
-                maxDistance = Math.max(maxDistance, neighbor.getDistance());
-            }
-        }
-        Histogram histogram = new Histogram(0, maxDistance, SPACING_HISTOGRAM_RESOLUTION);
-        AngleFilter filter = new AngleFilter(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
-        for (Character component : components) {
-            for (Neighbor neighbor : component.getNeighbors()) {
-                if (filter.matches(neighbor)) {
-                    histogram.add(neighbor.getDistance());
-                }
-            }
-        }
-
-        histogram.gaussianSmooth(SPACING_HISTOGRAM_SMOOTHING_LENGTH, SPACING_HIST_SMOOTHING_STANDARD_DEVIATION);
-        return histogram.getPeakValue();
-    }
-
-}
@@ -1,90 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-
-@Service
-public class ZoneBuilderService {
-
-    private static final double MIN_HORIZONTAL_DISTANCE_MULTIPLIER = -0.5;
-    private static final double MAX_VERTICAL_DISTANCE_MULTIPLIER = 1.2;
-
-    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -3.0;
-
-    private static final double MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER = 0.5;
-
-    private static final double MIN_LINE_SIZE_SCALE = 0.9;
-
-    private static final double MAX_LINE_SIZE_SCALE = 2.5;
-
-    private static final double ANGLE_TOLERANCE = Math.PI / 6;
-
-    public static final int MAX_ZONES = 300;
-
-
-    public List<Zone> buildZones(List<Line> lines, double characterSpacing, double lineSpacing) {
-
-        double minHorizontalDistance = characterSpacing * MIN_HORIZONTAL_DISTANCE_MULTIPLIER;
-        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_DISTANCE_MULTIPLIER;
-        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
-        double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;
-
-        DisjointSets<Line> sets = new DisjointSets<>(lines);
-
-        double meanHeight = calculateMeanHeight(lines);
-
-        lines.forEach(outerLine -> //
-                lines.forEach(innerLine -> {
-
-                    double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
-                    scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
-
-                    if (!sets.areTogether(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {
-
-                        double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
-                        double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
-
-                        if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance //
-                                || minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
-                            sets.union(outerLine, innerLine);
-                        }
-                    }
-                }));
-
-        List<Zone> zones = new ArrayList<>();
-        sets.forEach(group -> {
-            zones.add(new Zone(new ArrayList<>(group)));
-        });
-
-        if (zones.size() > MAX_ZONES) {
-            List<Line> oneZoneLines = new ArrayList<>();
-            for (Zone zone : zones) {
-                oneZoneLines.addAll(zone.getLines());
-            }
-            return List.of(new Zone(oneZoneLines));
-        }
-
-        return zones;
-    }
-
-
-    private double calculateMeanHeight(List<Line> lines) {
-
-        double meanHeight = 0.0;
-        double weights = 0.0;
-        for (Line line : lines) {
-            double weight = line.getLength();
-            meanHeight += line.getHeight() * weight;
-            weights += weight;
-        }
-        meanHeight /= weights;
-        return meanHeight;
-    }
-
-}
@@ -1,18 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils;
-
-public class DoubleUtils {
-
-    public static int compareDouble(double d1, double d2, double precision) {
-
-        if (Double.isNaN(d1) || Double.isNaN(d2)) {
-            return Double.compare(d1, d2);
-        }
-        if (precision == 0) {
-            precision = 1;
-        }
-        long i1 = Math.round(d1 / precision);
-        long i2 = Math.round(d2 / precision);
-        return Long.valueOf(i1).compareTo(i2);
-    }
-
-}
@@ -18,8 +18,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
@@ -31,6 +29,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

@@ -8,10 +8,10 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Objects;

+import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;

 import lombok.experimental.UtilityClass;

@@ -110,6 +110,7 @@ public class SearchTextWithTextPositionFactory {
        return context.stringIdx - context.lastHyphenIdx < MAX_HYPHEN_LINEBREAK_DISTANCE;
    }

+
    private static List<Boundary> mergeToBoundaries(List<Integer> integers) {

        if (integers.isEmpty()) {
@@ -125,8 +126,9 @@ public class SearchTextWithTextPositionFactory {
            }
            end = current + 1;
        }
-        if (boundaries.isEmpty())
+        if (boundaries.isEmpty()) {
            boundaries.add(new Boundary(start, end));
+        }
        return boundaries;
    }

@@ -138,6 +140,7 @@ public class SearchTextWithTextPositionFactory {
        }
    }

+
    private boolean isLineBreak(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition) {

        return Objects.equals(currentTextPosition.getUnicode(), "\n") || isDeltaYLargerThanTextHeight(currentTextPosition, previousTextPosition);
@@ -163,17 +166,7 @@ public class SearchTextWithTextPositionFactory {

    private boolean isHyphen(String unicodeCharacter) {

-        return Objects.equals(unicodeCharacter, "-") || //
-                Objects.equals(unicodeCharacter, "~") || //
-                Objects.equals(unicodeCharacter, "‐") || //
-                Objects.equals(unicodeCharacter, "‒") || //
-                Objects.equals(unicodeCharacter, "⁻") || //
-                Objects.equals(unicodeCharacter, "−") || //
-                Objects.equals(unicodeCharacter, "﹣") || //
-                Objects.equals(unicodeCharacter, "゠") || //
-                Objects.equals(unicodeCharacter, "⁓") || //
-                Objects.equals(unicodeCharacter, "‑") || //
-                Objects.equals(unicodeCharacter, "\u00AD");
+        return false;
    }


@@ -11,12 +11,12 @@ import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TableMergingUtility;

 import lombok.experimental.UtilityClass;
@@ -8,15 +8,15 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import lombok.experimental.UtilityClass;
@@ -2,10 +2,10 @@ package com.knecon.fforesight.service.layoutparser.processor.services.factory;

 import java.util.List;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -7,11 +7,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -9,7 +8,6 @@ import java.util.Map;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;

@@ -329,6 +329,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
    }

+
    @Override
    public String getText(PDDocument doc) throws IOException {

@@ -25,10 +25,23 @@ import java.io.StringWriter;
 import java.io.Writer;
 import java.text.Bidi;
 import java.text.Normalizer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.regex.Pattern;

-import lombok.Getter;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.pdfbox.cos.COSDictionary;
@@ -46,6 +59,8 @@ import org.apache.pdfbox.text.TextPositionComparator;

 import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;

+import lombok.Getter;
+
 /**
 * This is just a copy except i only adjusted lines 594-607 cause this is a bug in Pdfbox.
 * see S416.pdf
@@ -194,40 +209,33 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
    }


+    public void beginMarkedContentSequence(COSName tag, COSDictionary properties) {

-    public void beginMarkedContentSequence(COSName tag, COSDictionary properties)
-    {
        PDMarkedContent markedContent = PDMarkedContent.create(tag, properties);
-        if (this.currentMarkedContents.isEmpty())
-        {
+        if (this.currentMarkedContents.isEmpty()) {
            this.markedContents.add(markedContent);
-        }
-        else
-        {
-            PDMarkedContent currentMarkedContent =
-                    this.currentMarkedContents.peek();
-            if (currentMarkedContent != null)
-            {
+        } else {
+            PDMarkedContent currentMarkedContent = this.currentMarkedContents.peek();
+            if (currentMarkedContent != null) {
                currentMarkedContent.addMarkedContent(markedContent);
            }
        }
        this.currentMarkedContents.push(markedContent);
    }

+
    @Override
-    public void endMarkedContentSequence()
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void endMarkedContentSequence() {
+
+        if (!this.currentMarkedContents.isEmpty()) {
            this.currentMarkedContents.pop();
        }
    }


-    public void xobject(PDXObject xobject)
-    {
-        if (!this.currentMarkedContents.isEmpty())
-        {
+    public void xobject(PDXObject xobject) {
+
+        if (!this.currentMarkedContents.isEmpty()) {
            this.currentMarkedContents.peek().addXObject(xobject);
        }
    }
@@ -635,7 +643,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                        var normalized = normalize(line);
 //                        normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent()

-
                        lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine);
                        writeLine(normalized, current.isParagraphStart);
                        line.clear();
@@ -914,8 +921,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                    textList.add(text);
                }
            }
-            if (!this.currentMarkedContents.isEmpty())
-            {
+            if (!this.currentMarkedContents.isEmpty()) {
                this.currentMarkedContents.peek().addText(text);
            }
        }
@@ -2102,7 +2108,9 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            return endParagraphWritten;
        }

-        public void setEndParagraphWritten(){
+
+        public void setEndParagraphWritten() {
+
            endParagraphWritten = true;
        }

@@ -2145,7 +2153,6 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
            this.isHangingIndent = true;
        }

-
    }

 }
@@ -1,11 +1,12 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.visualization;

+import static com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService.LINE_WIDTH;
+
 import java.awt.Color;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
-import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -23,59 +24,25 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.ColoredLine;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.ColoredRectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.LayoutGrid;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-import com.knecon.fforesight.service.viewerdoc.ContentStreams;
-import com.knecon.fforesight.service.viewerdoc.model.ColoredLine;
-import com.knecon.fforesight.service.viewerdoc.model.ColoredRectangle;
-import com.knecon.fforesight.service.viewerdoc.model.LayoutGrid;
-import com.knecon.fforesight.service.viewerdoc.model.PlacedText;
-import com.knecon.fforesight.service.viewerdoc.model.Standard14EmbeddableFont;
-import com.knecon.fforesight.service.viewerdoc.model.Visualizations;
-import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
-
-import io.micrometer.observation.annotation.Observed;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;

@Service
-@RequiredArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
 public class LayoutGridService {

-    ViewerDocumentService viewerDocumentService;
-
-    static float FONT_SIZE = 10f;
-    static float LINE_WIDTH = 1f;
-    static Standard14EmbeddableFont FONT = Standard14EmbeddableFont.helvetica();
-
-    static Color INNER_LINES_COLOR = new Color(255, 175, 175);
-    static Color PARAGRAPH_COLOR = new Color(70, 130, 180);
-    static Color TABLE_COLOR = new Color(102, 205, 170);
-    static Color SECTION_COLOR = new Color(50, 50, 50);
-    static Color HEADLINE_COLOR = new Color(162, 56, 56);
-    static Color HEADER_COLOR = new Color(171, 131, 6);
-    static Color IMAGE_COLOR = new Color(253, 63, 146);
+    private static final Color INNER_LINES_COLOR = new Color(255, 175, 175);
+    private static final Color PARAGRAPH_COLOR = new Color(70, 130, 180);
+    public static final Color TABLE_COLOR = new Color(102, 205, 170);
+    public static final Color SECTION_COLOR = new Color(50, 50, 50);
+    public static final Color HEADLINE_COLOR = new Color(162, 56, 56);
+    public static final Color HEADER_COLOR = new Color(171, 131, 6);
+    public static final Color IMAGE_COLOR = new Color(253, 63, 146);


-    @SneakyThrows
-    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
-    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {
-
-        LayoutGrid layoutGrid = createLayoutGrid(document);
-
-        viewerDocumentService.addVisualizationsOnPage(originFile,
-                destinationFile,
-                Visualizations.builder()
-                        .layer(ContentStreams.KNECON_LAYOUT)
-                        .visualizationsOnPages(layoutGrid.getVisualizationsPerPages())
-                        .layerVisibilityDefaultValue(layerVisibilityDefaultValue)
-                        .build());
-    }
-
-
-    private LayoutGrid createLayoutGrid(Document document) {
+    public LayoutGrid createLayoutGrid(Document document) {

        LayoutGrid layoutGrid = new LayoutGrid(document.getNumberOfPages());
        document.streamAllSubNodes().forEach(semanticNode -> {
@@ -136,11 +103,11 @@ public class LayoutGridService {
            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredLines();
            xs.forEach(x -> {
                Line2D line = new Line2D.Double(new Point2D.Double(x, tableBBox.getMaxY()), new Point2D.Double(x, tableBBox.getMinY()));
-                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR, LINE_WIDTH));
+                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR));
            });
            ys.forEach(y -> {
                Line2D line = new Line2D.Double(new Point2D.Double(tableBBox.getMinX(), y), new Point2D.Double(tableBBox.getMaxX(), y));
-                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR, LINE_WIDTH));
+                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR));
            });
        }
    }
@@ -168,9 +135,9 @@ public class LayoutGridService {
            List<Line2D> lines = createLinesFromRectangle(r, firstPage.getRotation());
            // add string to top line
            var firstLine = lines.remove(0);
-            coloredLines.add(new ColoredLine(firstLine, color, LINE_WIDTH));
+            coloredLines.add(new ColoredLine(firstLine, color));
            for (Line2D line : lines) {
-                coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+                coloredLines.add(new ColoredLine(line, color));
            }
            return;
        }
@@ -185,7 +152,6 @@ public class LayoutGridService {
    }


-    @SneakyThrows
    private void addPlacedText(Page page, Rectangle2D textBBox, String s, LayoutGrid layoutGrid) {

        Point2D.Float upperLeftCorner = switch (page.getRotation()) {
@@ -195,8 +161,7 @@ public class LayoutGridService {
            default -> new Point2D.Float((float) (textBBox.getMinX()), (float) textBBox.getMaxY());
        };
        var placedTexts = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getPlacedTexts();
-        upperLeftCorner.setLocation(upperLeftCorner.getX() - ((FONT.getStringWidth(s) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), upperLeftCorner.getY() - FONT_SIZE);
-        placedTexts.add(PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, Color.BLACK, FONT));
+        placedTexts.add(new PlacedText(s, upperLeftCorner));
    }


@@ -211,9 +176,9 @@ public class LayoutGridService {
        midPageLines.remove(1);
        // add string to left line
        var leftLine = midPageLines.remove(1);
-        coloredLines.add(new ColoredLine(leftLine, color, LINE_WIDTH));
+        coloredLines.add(new ColoredLine(leftLine, color));
        for (Line2D line : midPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+            coloredLines.add(new ColoredLine(line, color));
        }
    }

@@ -227,9 +192,9 @@ public class LayoutGridService {
        lastPageLines.remove(0);
        // add string to left line
        var leftLine = lastPageLines.remove(2);
-        coloredLines.add(new ColoredLine(leftLine, color, LINE_WIDTH));
+        coloredLines.add(new ColoredLine(leftLine, color));
        for (Line2D line : lastPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+            coloredLines.add(new ColoredLine(line, color));
        }
    }

@@ -243,9 +208,9 @@ public class LayoutGridService {
        firstPageLines.remove(2);
        // add string to top line
        var firstLine = firstPageLines.remove(0);
-        coloredLines.add(new ColoredLine(firstLine, color, LINE_WIDTH));
+        coloredLines.add(new ColoredLine(firstLine, color));
        for (Line2D line : firstPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+            coloredLines.add(new ColoredLine(line, color));
        }
    }

@@ -311,10 +276,7 @@ public class LayoutGridService {
    private void addAsRectangle(SemanticNode semanticNode, LayoutGrid layoutGrid, Color color) {

        semanticNode.getBBox()
-                .forEach((page, textBBox) -> layoutGrid.getVisualizationsPerPages()
-                        .get(page.getNumber() - 1)
-                        .getColoredRectangles()
-                        .add(new ColoredRectangle(textBBox, color, LINE_WIDTH)));
+                .forEach((page, textBBox) -> layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredRectangles().add(new ColoredRectangle(textBBox, color)));
    }

 }
@@ -1,270 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
-
-import java.awt.Color;
-import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.PDPageContentStream;
-import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
-import org.apache.pdfbox.util.Matrix;
-import org.springframework.core.io.ClassPathResource;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class PdfDraw {
-
-    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) throws IOException {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
-                        pageNumber,
-                        rectanglesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-
-    }
-
-
-    public static void drawRectanglesPerPageNumberedByLine(String filename, List<List<List<Rectangle2D>>> rectanglesPerPage, String tmpFileName) throws IOException {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                var rectanglesOnPage = rectanglesPerPage.get(pageNumber - 1);
-                for (int lineNumber = 0; lineNumber < rectanglesOnPage.size(); lineNumber++) {
-                    var rectanglesInLine = rectanglesOnPage.get(lineNumber);
-                    PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, rectanglesInLine, PdfVisualisationUtility.Options.builder().stroke(true).build());
-                    double y = Math.min(rectanglesInLine.get(0).getMinY(), rectanglesInLine.get(0).getMaxY());
-                    PdfVisualisationUtility.drawText(String.format("%d", lineNumber),
-                            pdDocument,
-                            new Point2D.Double(rectanglesInLine.get(0).getX() - (5 + (5 * countNumberOfDigits(lineNumber))), y + 2),
-                            pageNumber,
-                            PdfVisualisationUtility.Options.builder().stroke(true).build());
-                }
-            }
-            pdDocument.save(out);
-        }
-
-    }
-
-
-    private static int countNumberOfDigits(int num) {
-
-        int final_num = num;
-        if (final_num == 0) {
-            return 1;
-        }
-        int count = 0;
-        for (; final_num != 0; final_num /= 10) {
-            count++;
-        }
-        return count;
-    }
-
-
-    public static void drawDocumentGraph(PDDocument document, Document documentGraph) {
-
-        documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry));
-    }
-
-
-    public static void drawNode(PDDocument document, DocumentTree.Entry entry) {
-
-        Options options = buildStandardOptionsForNodes(entry);
-
-        drawBBoxAndLabelAndNumberOnPage(document, entry, options);
-
-    }
-
-
-    public static void drawTextBlock(PDDocument document, TextBlock textBlock, Options options) {
-
-        textBlock.getAtomicTextBlocks().forEach(atb -> drawAtomicTextBlock(document, atb, options));
-    }
-
-
-    public static void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, Options options) {
-
-        drawRectangle2DList(document, atomicTextBlock.getPage().getNumber(), atomicTextBlock.getPositions().stream().toList(), options);
-
-    }
-
-
-    @SneakyThrows
-    private static void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, Options options, boolean rotate) {
-
-        var pdPage = document.getPage(pageNumber - 1);
-        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
-
-        contentStream.setNonStrokingColor(options.getStrokeColor());
-        contentStream.setLineWidth(options.getStrokeWidth());
-
-        contentStream.beginText();
-        if (rotate) {
-            contentStream.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(15), (float) location.getX(), (float) location.getY()));
-        } else {
-            contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
-        }
-        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
-        contentStream.showText(string);
-        contentStream.endText();
-        contentStream.close();
-    }
-
-
-    @SneakyThrows
-    public static void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, Options options) {
-
-        var pdPage = document.getPage(pageNumber - 1);
-        drawRectangle2DList(document, rectCollection, options, pdPage);
-    }
-
-
-    private static void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, Options options, PDPage pdPage) throws IOException {
-
-        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
-
-        contentStream.setStrokingColor(options.getStrokeColor());
-        contentStream.setNonStrokingColor(options.getFillColor());
-        contentStream.setLineWidth(options.getStrokeWidth());
-
-        for (var r : rectCollection) {
-            contentStream.addRect((float) r.getMinX(), (float) r.getMinY(), (float) r.getWidth(), (float) r.getHeight());
-
-            if (options.isStroke() && options.isFill()) {
-                contentStream.fillAndStroke();
-            } else if (options.isStroke()) {
-                contentStream.stroke();
-            } else if (options.isFill()) {
-                contentStream.fill();
-            }
-        }
-        contentStream.close();
-    }
-
-
-    @SneakyThrows
-    public static void drawRectanglesAndLinesPerPage(String filename, List<List<Rectangle2D>> list, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-//                PdfVisualisationUtility.drawLine2DList(pdDocument,
-//                        pageNumber,
-//                        list.get(pageNumber - 1),
-//                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
-                        pageNumber,
-                        rectanglesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-    }
-
-
-    @SneakyThrows
-    public static void drawLinesPerPage(String filename, List<List<Ruling>> linesPerPage, String tmpFileName) {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                PdfVisualisationUtility.drawLine2DList(pdDocument,
-                        pageNumber,
-                        linesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-    }
-
-
-    @Builder
-    @AllArgsConstructor
-    @Getter
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    public static class Options {
-
-        boolean stroke;
-        @Builder.Default
-        Color strokeColor = Color.BLACK;
-        @Builder.Default
-        float strokeWidth = 1f;
-
-        boolean fill;
-        @Builder.Default
-        Color fillColor = Color.BLACK;
-
-    }
-
-
-    private static Options buildStandardOptionsForNodes(DocumentTree.Entry entry) {
-
-        return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
-            case DOCUMENT -> Color.LIGHT_GRAY;
-            case HEADER, FOOTER -> Color.GREEN;
-            case PARAGRAPH -> Color.BLUE;
-            case HEADLINE -> Color.RED;
-            case SECTION -> Color.BLACK;
-            case TABLE -> Color.ORANGE;
-            case TABLE_CELL -> Color.GRAY;
-            case IMAGE -> Color.MAGENTA;
-        }).build();
-    }
-
-
-    private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, Options options) {
-
-        Map<Page, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
-        for (Page page : rectanglesPerPage.keySet()) {
-            Rectangle2D rectangle2D = rectanglesPerPage.get(page);
-            if (entry.getType() == NodeType.SECTION) {
-                rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
-            }
-            drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options);
-            drawText(buildString(entry),
-                    document,
-                    new Point2D.Double(rectangle2D.getMinX(), rectangle2D.getMaxY() + 2),
-                    page.getNumber(),
-                    options,
-                    entry.getType() == NodeType.TABLE_CELL);
-        }
-    }
-
-
-    private static String buildString(DocumentTree.Entry entry) {
-
-        return entry.getNode().getNumberOnPage() + ": " + entry.getTreeId() + ": " + entry.getType();
-    }
-
-}
@@ -0,0 +1,168 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
+
+import java.awt.Color;
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Rectangle2D;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.pdfbox.cos.COSDictionary;
+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.PDResources;
+import org.apache.pdfbox.pdmodel.font.PDFont;
+import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
+import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentGroup;
+import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentProperties;
+import org.apache.pdfbox.pdmodel.graphics.state.PDExtendedGraphicsState;
+import org.apache.pdfbox.util.Matrix;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.ColoredLine;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.ColoredRectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.FilledRectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.LayoutGrid;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
+import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorCells;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ViewerDocumentService {
+
+    private static final String LAYER_NAME = "Layout grid";
+    private static final int FONT_SIZE = 10;
+    public static final float LINE_WIDTH = 1f;
+
+    private final LayoutGridService layoutGridService;
+
+
+    @SneakyThrows
+    public void createViewerDocument(PDDocument pdDocument,
+                                     Document document,
+                                     OutputStream outputStream,
+                                     Map<Integer,List<TableExtractorCells>> extractedTableCells,
+                                     boolean layerVisibilityDefaultValue) {
+
+        LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
+        // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
+        // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
+        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
+        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
+        PDOptionalContentGroup visualLayoutParsingLayer = addLayerToDocument(pdDocument, dictionariesToUpdate, true);
+        PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);
+
+        for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
+            PDPage pdPage = pdDocument.getPage(pageNumber);
+
+            AffineTransform textDeRotationMatrix = getTextDeRotationTransform(pdPage);
+            addLayerToPageRessources(pdPage);
+
+            // We need to save the graphics state before, such that our appended content cannot be affected by previous content streams with side effects,
+            // e.g. not escaped matrix transformations.
+            escapePreviousContents(pdDocument, pdPage);
+
+            VisualizationsOnPage visualizationsOnPage = layoutGrid.getVisualizationsPerPages().get(pageNumber);
+            assert pageNumber == visualizationsOnPage.getPageNumber();
+            // We need to append to the content stream, otherwise the content could be overlapped by following content.
+            try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true)) {
+                contentStream.beginMarkedContent(COSName.OC, visualLayoutParsingLayer);
+                contentStream.saveGraphicsState();
+
+                contentStream.setLineWidth(LINE_WIDTH);
+                for (TableExtractorCells tableCells : extractedTableCells.get(pageNumber)) {
+                    contentStream.setStrokingColor(new Color(0xB700FF));
+                    contentStream.addRect((float) tableCells.getX0(), (float) tableCells.getY0(), (float) tableCells.getWidth(), (float) tableCells.getHeight());
+                    contentStream.stroke();
+                    contentStream.setFont(font, FONT_SIZE);
+                    contentStream.beginText();
+                    Matrix textMatrix = new Matrix((float) textDeRotationMatrix.getScaleX(),
+                            (float) textDeRotationMatrix.getShearX(),
+                            (float) textDeRotationMatrix.getShearY(),
+                            (float) textDeRotationMatrix.getScaleY(),
+                            tableCells.getX0() ,
+                            tableCells.getY0());
+                    textMatrix.translate(-((font.getStringWidth(tableCells.getLabel()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
+                    contentStream.setTextMatrix(textMatrix);
+                    contentStream.showText(tableCells.getLabel());
+                    contentStream.endText();
+                }
+                contentStream.restoreGraphicsState();
+                contentStream.endMarkedContent();
+            }
+            dictionariesToUpdate.add(pdPage.getCOSObject());
+            dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
+        }
+        dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
+//        dictionariesToUpdate.add(pdDocument.getDocument().getTrailer());
+        pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
+    }
+
+
+    private static void addLayerToPageRessources(PDPage pdPage) {
+
+        PDResources resources = pdPage.getResources();
+        if (resources == null) {
+            resources = new PDResources();
+            pdPage.setResources(resources);
+        }
+    }
+
+
+    private static void escapePreviousContents(PDDocument pdDocument, PDPage pdPage) throws IOException {
+
+        try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.PREPEND, false)) {
+            contentStream.saveGraphicsState();
+        }
+        try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, false)) {
+            contentStream.restoreGraphicsState();
+        }
+    }
+
+
+    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate, boolean layerVisibilityDefaultValue) {
+
+        PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
+        PDOptionalContentProperties ocprops = catalog.getOCProperties();
+        if (ocprops == null) {
+            ocprops = new PDOptionalContentProperties();
+            catalog.setOCProperties(ocprops);
+        }
+        PDOptionalContentGroup layer = null;
+        if (ocprops.hasGroup(LAYER_NAME)) {
+            layer = ocprops.getGroup(LAYER_NAME);
+        } else {
+            layer = new PDOptionalContentGroup(LAYER_NAME);
+            ocprops.addGroup(layer);
+        }
+        ocprops.setGroupEnabled(layer, layerVisibilityDefaultValue);
+        dictionariesToUpdate.add(catalog.getCOSObject());
+        return layer;
+    }
+
+
+    private static AffineTransform getTextDeRotationTransform(PDPage page) {
+
+        return AffineTransform.getQuadrantRotateInstance(switch (page.getRotation()) {
+            case 90 -> 3;
+            case 180 -> 2;
+            case 270 -> 1;
+            default -> 0;
+        });
+    }
+
+}
@@ -1,12 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;

-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import lombok.experimental.UtilityClass;
-import org.apache.pdfbox.cos.COSName;
-import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
-import org.apache.pdfbox.text.TextPosition;
-
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.Collections;
@@ -14,12 +7,22 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;

+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
+import org.apache.pdfbox.text.TextPosition;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+
+import lombok.experimental.UtilityClass;
+
@UtilityClass
 public class MarkedContentUtils {

    public static final String HEADER = "Header";
    public static final String FOOTER = "Footer";

+
    public List<Rectangle2D> getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {

        if (markedContents == null) {
@@ -31,7 +34,8 @@ public class MarkedContentUtils {
                .filter(m -> m.getProperties() != null)
                .filter(m -> m.getProperties().getItem("Subtype") != null)
                .filter(m -> ((COSName) m.getProperties().getItem("Subtype")).getName().equals(subtype))
-                .map(PDMarkedContent::getContents).flatMap(Collection::stream)
+                .map(PDMarkedContent::getContents)
+                .flatMap(Collection::stream)
                .filter(t -> t instanceof TextPosition)
                .map(t -> (TextPosition) t)
                .filter(t -> !t.getUnicode().equals(" "))
@@ -41,16 +45,19 @@ public class MarkedContentUtils {
            return Collections.emptyList();
        }

-        return markedContentByYPosition.values().stream()
-                .map(textPositions -> new TextPositionSequence(textPositions.stream()
-                        .toList(), 0, true)
-                        .getRectangle())
-                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight()))).collect(Collectors.toList());
+        return markedContentByYPosition.values()
+                .stream()
+                .map(textPositions -> new TextPositionSequence(textPositions.stream().toList(), 0, true).getRectangle())
+                .map(t -> new Rectangle2D.Float(t.getTopLeft().getX(), t.getTopLeft().getY() - Math.abs(t.getHeight()), t.getWidth(), Math.abs(t.getHeight())))
+                .collect(Collectors.toList());
    }


    public boolean intersects(TextPageBlock textBlock, Map<String, List<Rectangle2D>> markedContentBboxPerType, String type) {
-        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type).stream().anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
+
+        return markedContentBboxPerType.get(type) != null && markedContentBboxPerType.get(type)
+                .stream()
+                .anyMatch(rectangle -> rectangle.intersects(textBlock.getPdfMinX(), textBlock.getPdfMinY(), textBlock.getWidth(), textBlock.getHeight()));
    }

 }
@@ -19,10 +19,9 @@ public final class PositionUtils {

        double threshold = textBlock.getMostPopularWordHeight() * 3;

-        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX()
-                && textBlock.getPdfMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth()
-                && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY()
-                && textBlock.getPdfMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
+        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX() && textBlock.getPdfMaxX() - threshold < btf.getTopLeft()
+                .getX() + btf.getWidth() && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY() && textBlock.getPdfMaxY() - threshold < btf.getTopLeft()
+                .getY() + btf.getHeight()) {
            return true;
        } else {
            return false;
@@ -41,11 +41,14 @@ public class RectangleTransformations {

        return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DBBoxCollector());
    }
+
+
    public static Collector<Rectangle2D, Rectangle2DBBoxCollector.BBox, Rectangle2D> collectBBox() {

        return new Rectangle2DBBoxCollector();
    }

+
    public static PDRectangle toPDRectangleBBox(List<Rectangle> rectangles) {

        Rectangle2D rectangle2D = RectangleTransformations.rectangleBBox(rectangles);
@@ -70,6 +73,7 @@ public class RectangleTransformations {
        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
    }

+
    public static Rectangle2D rectangleBBox(List<Rectangle> rectangles) {

        return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DBBoxCollector());
@@ -84,6 +88,7 @@ public class RectangleTransformations {
                -redactionLogRectangle.getHeight());
    }

+
    public static Rectangle2D toRectangle2D(PDRectangle rectangle) {

        return new Rectangle2D.Double(rectangle.getLowerLeftX(), rectangle.getLowerLeftY(), rectangle.getWidth(), rectangle.getHeight());
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 import java.util.List;
 import java.util.stream.Collectors;

-
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

@@ -28,15 +28,13 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPosit
 *
 * @author Ben Litchfield
 */
-public class TextPositionSequenceComparator implements Comparator<TextPositionSequence>
-{
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
+
    @Override
-    public int compare(TextPositionSequence pos1, TextPositionSequence pos2)
-    {
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
        // only compare text that is in the same direction
        int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
-        if (cmp1 != 0)
-        {
+        if (cmp1 != 0) {
            return cmp1;
        }

@@ -54,19 +52,13 @@ public class TextPositionSequenceComparator implements Comparator<TextPositionSe
        float yDifference = Math.abs(pos1YBottom - pos2YBottom);

        // we will do a simple tolerance comparison
-        if (yDifference < .1 ||
-                pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
-                pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
-        {
+        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
            return Float.compare(x1, x2);
-        }
-        else if (pos1YBottom < pos2YBottom)
-        {
+        } else if (pos1YBottom < pos2YBottom) {
            return -1;
-        }
-        else
-        {
+        } else {
            return 1;
        }
    }
+
 }
@@ -3,53 +3,40 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
    application
    id("com.knecon.fforesight.java-conventions")
-    id("org.springframework.boot") version "3.1.5"
+    id("org.springframework.boot") version "3.1.3"
    id("io.spring.dependency-management") version "1.1.3"
    id("org.sonarqube") version "4.3.0.3225"
-    id("io.freefair.lombok") version "8.4"
+    id("io.freefair.lombok") version "8.2.2"
 //    id("org.graalvm.buildtools.native") version "0.9.23"
 }

 description = "layoutparser-service-server"

-configurations {
-    all {
-        exclude(group = "commons-logging", module = "commons-logging")
-        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
-        exclude(group = "com.iqser.red.commons", module = "logging-commons")
-    }
-}
-
-
 val jacksonVersion = "2.15.2"
 val pdfBoxVersion = "3.0.0"
-val springBootStarterVersion = "3.1.5"

 dependencies {
    implementation(project(":layoutparser-service-processor"))
    implementation(project(":layoutparser-service-internal-api"))

    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
-    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
-    implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
+    implementation("com.knecon.fforesight:tenant-commons:0.14.0")
+    implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
    implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
-    implementation("net.logstash.logback:logstash-logback-encoder:7.4")

    // for integration testing only
-    testImplementation(project(":viewer-doc-processor"))
    testImplementation(project(":layoutparser-service-internal-api"))

-    testImplementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
-    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")
    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.144.0")
    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
    testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
    testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.3")
 }

 /*
@@ -5,7 +5,6 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
@@ -14,9 +13,6 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingService
 import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.aop.ObservedAspect;
-
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class, MessagingConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
@@ -27,11 +23,4 @@ public class Application {
        SpringApplication.run(Application.class, args);
    }

-
-    @Bean
-    public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
-
-        return new ObservedAspect(observationRegistry);
-    }
-
 }
@@ -24,7 +24,8 @@ public class MessagingConfiguration {
    @Bean
    public Queue layoutparsingResponseQueue() {

-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
    }


@@ -5,19 +5,10 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/

 fforesight.tenants.remote: true

-logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
-
-logging.type: ${LOGGING_TYPE:CONSOLE}
-kubernetes.namespace: ${NAMESPACE:default}
-project.version: 1.0-SNAPSHOT
-
-
 server:
  port: 8080

 spring:
-  application:
-    name: layoutparser-service
  main:
    allow-circular-references: true # FIXME
  rabbitmq:
@@ -41,13 +32,6 @@ management:
    prometheus.enabled: ${monitoring.enabled:false}
    health.enabled: true
  endpoints.web.exposure.include: prometheus, health
-  tracing:
-    enabled: ${TRACING_ENABLED:false}
-    sampling:
-      probability: ${TRACING_PROBABILITY:1.0}
-  otlp:
-    tracing:
-      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}


 storage:
@@ -1,17 +0,0 @@
-<configuration>
-
-    <springProperty scope="configuration" name="logType" source="logging.type"/>
-    <springProperty scope="context" name="application.name" source="spring.application.name"/>
-    <springProperty scope="context" name="version" source="project.version"/>
-    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
-    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
-
-    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
-    </appender>
-
-    <root level="INFO">
-        <appender-ref ref="${logType}"/>
-    </root>
-
-</configuration>
@@ -26,6 +26,7 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
@@ -46,11 +47,12 @@ public class BdrJsonBuildTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(File file) {

-        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
-                file,
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                file.toString()));
+        try (PDDocument pdDocument = Loader.loadPDF(file)) {
+            return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
+                    pdDocument,
+                    new ImageServiceResponse(),
+                    new TableServiceResponse(), new TableExtractorResponse()));
+        }
    }


@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

+import org.apache.pdfbox.Loader;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -32,6 +33,7 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableExtractorResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
@@ -95,10 +97,9 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                pdfFileResource.getFile(),
+                Loader.loadPDF(pdfFileResource.getFile()),
                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filePath));
+                new TableServiceResponse(), new TableExtractorResponse()));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -148,7 +149,6 @@ public class HeadlinesGoldStandardIntegrationTest {
    @EqualsAndHashCode
    @AllArgsConstructor
    @ToString
-    @SuppressWarnings("PMD.AvoidFieldNameMatchingTypeName")
    private class Headline {

        private int page;
@@ -25,10 +25,8 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
    @SneakyThrows
    public void testLayoutParserEndToEnd() {

-        String s = "（";
-        String s1 = "）";
-        prepareStorage("files/Minimal Examples/WrongOrderPage1.pdf");
-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        prepareStorage("files/bdr/btd_19_053_1905391.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.TAAS);
        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
    }
@@ -6,7 +6,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;

-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -18,9 +17,7 @@ import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest
 import lombok.SneakyThrows;

 public class DocumentDataTests extends BuildDocumentTest {
-
    @Test
-    @Disabled // This test takes waaaaaay too long, it's ridiculous
    @SneakyThrows
    public void createDocumentDataForAllFiles() {

@@ -39,12 +36,11 @@ public class DocumentDataTests extends BuildDocumentTest {
        for (String pdfFileName : pdfFileNames) {
            System.out.println(pdfFileName);
            DocumentData documentData = DocumentDataMapper.toDocumentData(buildGraph(resource.getFile().toPath().getParent().relativize(Path.of(pdfFileName)).toString()));
-            File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
+                        File outputFile = Path.of(outPath).resolve(resource.getFile().toPath().relativize(Path.of(pdfFileName))).toFile();
            outputFile.toPath().getParent().toFile().mkdirs();
            try (var out = new FileOutputStream(outputFile.toString().replace(".pdf", ".json"))) {
                ObjectMapperFactory.create().writeValue(out, documentData);
            }
        }
    }
-
 }
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
yhampe	aa1e77dda3	RED-7375 table extractor prototype	2024-02-01 08:23:53 +01:00
yhampe	0c3b910088	RED-7375 table extractor prototype	2024-01-30 11:59:18 +01:00
yhampe	b4e5f2da2f	RED-7375 table extractor prototype added label changed colour of bounding boxes	2024-01-17 16:58:53 +01:00
yhampe	b47b187c8a	RED-7375: table extractor prototype added layer for table extractor results in ViewerDocumentService	2024-01-15 09:43:23 +01:00
yhampe	0d4800622d	RED-7375:Integrate Table Parsing Service * integrated table parsing service * for visualization simpy drawing the extracted table in ViewerDocumentService	2023-12-01 13:26:16 +01:00
yhampe	cc492bc50d	RED-7375:Integrate Table Parsing Service * fixed merging mistake	2023-11-29 10:06:05 +01:00
yhampe	1c3c5385e1	Merge remote-tracking branch 'origin/RED-7375' into RED-7375	2023-11-29 09:40:24 +01:00
yhampe	bce2558133	RED-7375:Integrate Table Parsing Service * fixed end2end test by introducing sample table extractor response	2023-11-29 09:39:58 +01:00
yhampe	2200574b6d	RED-7375:Integrate Table Parsing Service * disabled not needed test	2023-11-29 09:39:58 +01:00
yhampe	775c943f7e	RED-7375: * using ViewerDocumentService to draw TableExtractorResponse into documents	2023-11-29 09:39:53 +01:00
yhampe	b7fe6fd3c4	RED-7375:Integrate Table Parsing Service * fixed end2end test by introducing sample table extractor response	2023-11-29 09:37:47 +01:00
yhampe	3dd447ebef	RED-7375:Integrate Table Parsing Service * disabled not needed test	2023-11-29 08:41:31 +01:00
yhampe	f4e93ef03b	RED-7375: * using ViewerDocumentService to draw TableExtractorResponse into documents	2023-11-28 15:14:04 +01:00