Merge branch 'hotfix' into 'main'

hotfix: double viewerdoc writes in rare cases lead to some contentstreams not being written See merge request fforesight/layout-parser!104
hotfix: double viewerdoc writes in rare cases lead to some contentstreams not being written
2024-02-26 12:32:14 +01:00 · 2024-02-26 12:24:15 +01:00 · 2024-02-23 14:08:57 +01:00 · 2024-02-23 14:02:53 +01:00 · 2024-02-23 13:55:11 +01:00 · 2024-02-23 12:46:51 +01:00
80 changed files with 1035 additions and 1914 deletions
@@ -5,6 +5,7 @@ import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Stream;

 import io.swagger.v3.oas.annotations.media.Schema;
@@ -115,6 +116,8 @@ public class DocumentStructure implements Serializable {
        Map<String, String> properties;
        @Schema(description = "All child Entries of this Entry.", example = "[1, 2, 3]")
        List<EntryData> children;
+        @Schema(description = "Describes the origin of the semantic node", example = "[ALGORITHM]")
+        Set<LayoutEngine> engines;


        @Override
@@ -0,0 +1,6 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+public enum LayoutEngine {
+    ALGORITHM,
+    AI
+}
@@ -18,11 +18,15 @@ public record LayoutParsingRequest(

        @Schema(description = "Path to the original PDF file.")//
        @NonNull String originFileStorageId,//
+
+
        @Schema(description = "Optional Path to the table extraction file.")//
        Optional<String> tablesFileStorageId,//
        @Schema(description = "Optional Path to the image classification file.")//
        Optional<String> imagesFileStorageId,//

+        @Schema(description = "Optional Path to the the visual layout parsing service file") Optional<String> visualLayoutParsingFileId,//
+
        @Schema(description = "Path where the Document Structure File will be stored.")//
        @NonNull String structureFileStorageId,//
        @Schema(description = "Path where the Research Data File will be stored.")//
@@ -10,7 +10,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;

 import org.apache.pdfbox.Loader;
@@ -27,7 +26,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
@@ -35,9 +33,11 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageB
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
@@ -49,7 +49,6 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
@@ -89,7 +88,7 @@ public class LayoutParsingPipeline {
    RedactManagerBlockificationService redactManagerBlockificationService;
    LayoutGridService layoutGridService;
    ObservationRegistry observationRegistry;
-    DocstrumSegmentationService docstrumSegmentationService;
+    VisualLayoutParsingAdapter visualLayoutParsingAdapter;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -98,23 +97,36 @@ public class LayoutParsingPipeline {
        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);
+        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId())
+                .orElse(originFile);
+
+        VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse();
+        if (layoutParsingRequest.visualLayoutParsingFileId()
+                .isPresent()) {
+            visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId()
+                                                                                                         .get());
+        }

        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
-        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
-            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
+        if (layoutParsingRequest.imagesFileStorageId()
+                .isPresent()) {
+            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId()
+                                                                                     .get());
        }

        TableServiceResponse tableServiceResponse = new TableServiceResponse();
-        if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
-            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
+        if (layoutParsingRequest.tablesFileStorageId()
+                .isPresent()) {
+            tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId()
+                                                                                     .get());
        }

        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
-                originFile,
-                imageServiceResponse,
-                tableServiceResponse,
-                layoutParsingRequest.identifier().toString());
+                                                                    originFile,
+                                                                    imageServiceResponse,
+                                                                    tableServiceResponse,
+                                                                    visualLayoutParsingResponse,
+                                                                    layoutParsingRequest.identifier().toString());

        log.info("Building document graph for {}", layoutParsingRequest.identifier());

@@ -146,25 +158,25 @@ public class LayoutParsingPipeline {
                .numberOfPages(documentGraph.getNumberOfPages())
                .duration(System.currentTimeMillis() - start)
                .message(format("""
-                                Layout parsing has finished in %.02f s.
-                                identifiers: %s
-                                %s
-                                Files have been saved with Ids:
-                                Structure: %s
-                                Text: %s
-                                Positions: %s
-                                PageData: %s
-                                Simplified Text: %s
-                                Viewer Doc: %s""",
-                        ((float) (System.currentTimeMillis() - start)) / 1000,
-                        layoutParsingRequest.identifier(),
-                        buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
-                        layoutParsingRequest.structureFileStorageId(),
-                        layoutParsingRequest.textBlockFileStorageId(),
-                        layoutParsingRequest.positionBlockFileStorageId(),
-                        layoutParsingRequest.pageFileStorageId(),
-                        layoutParsingRequest.simplifiedTextStorageId(),
-                        layoutParsingRequest.viewerDocumentStorageId()))
+                                        Layout parsing has finished in %.02f s.
+                                        identifiers: %s
+                                        %s
+                                        Files have been saved with Ids:
+                                        Structure: %s
+                                        Text: %s
+                                        Positions: %s
+                                        PageData: %s
+                                        Simplified Text: %s
+                                        Viewer Doc: %s""",
+                                ((float) (System.currentTimeMillis() - start)) / 1000,
+                                layoutParsingRequest.identifier(),
+                                buildSemanticNodeCountMessage(documentGraph.getNumberOfPages(), documentGraph.buildSemanticNodeCounts()),
+                                layoutParsingRequest.structureFileStorageId(),
+                                layoutParsingRequest.textBlockFileStorageId(),
+                                layoutParsingRequest.positionBlockFileStorageId(),
+                                layoutParsingRequest.pageFileStorageId(),
+                                layoutParsingRequest.simplifiedTextStorageId(),
+                                layoutParsingRequest.viewerDocumentStorageId()))
                .build();

    }
@@ -174,9 +186,9 @@ public class LayoutParsingPipeline {

        AtomicReference<Document> documentReference = new AtomicReference<>();

-        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
-            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
-        });
+        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry)
+                .contextualName("build-document-graph")
+                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument)));

        return documentReference.get();
    }
@@ -185,14 +197,14 @@ public class LayoutParsingPipeline {
    private String buildSemanticNodeCountMessage(int numberOfPages, Map<NodeType, Long> semanticNodeCounts) {

        return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
-                numberOfPages,
-                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
-                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
-                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
-                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
-                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
-                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
-                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
+                             numberOfPages,
+                             semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
+                             semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
+                             semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
+                             semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
+                             semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
+                             semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
+                             semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
    }


@@ -202,13 +214,14 @@ public class LayoutParsingPipeline {
                                              File originFile,
                                              ImageServiceResponse imageServiceResponse,
                                              TableServiceResponse tableServiceResponse,
+                                              VisualLayoutParsingResponse visualLayoutParsingResponse,
                                              String identifier) {

        PDDocument originDocument = openDocument(originFile);
        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
-
+        Map<Integer, List<ClassifiedImage>> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse);
        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

@@ -247,37 +260,11 @@ public class LayoutParsingPipeline {
            PDRectangle cropbox = pdPage.getCropBox();
            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());

-            // Docstrum
-            AtomicInteger num = new AtomicInteger(pageNumber);
-            var zones = docstrumSegmentationService.segmentPage(stripper.getTextPositionSequences());
-
-            List<AbstractPageBlock> pageBlocks = new ArrayList<>();
-            AtomicInteger numOnPage = new AtomicInteger(1);
-//            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-            zones.forEach(zone -> {
-
-                List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-                zone.getLines().forEach(line -> {
-                    line.getWords().forEach(word -> {
-                        textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), num.get()));
-                    });
-                });
-
-                var cps = redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-                cps.getTextBlocks().forEach(cp -> {
-                    pageBlocks.add(redactManagerBlockificationService.buildTextBlock(((TextPageBlock) cp).getSequences(), numOnPage.getAndIncrement()));
-                });
-            });
-
-//            ClassificationPage classificationPage = switch (layoutParsingType) {
-//                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//                case TAAS -> taasBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//                case DOCUMINE -> docuMineBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//            };
-
-            ClassificationPage classificationPage = new ClassificationPage(pageBlocks);
-
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            };
            classificationPage.setCleanRulings(cleanRulings);
            classificationPage.setRotation(rotation);
            classificationPage.setLandscape(isLandscape);
@@ -294,6 +281,14 @@ public class LayoutParsingPipeline {
                imageServiceResponseAdapter.findOcr(classificationPage);
            }

+            if (signatures.containsKey(pageNumber)) {
+                if (classificationPage.getImages() == null || classificationPage.getImages().size() == 0) {
+                    classificationPage.setImages(signatures.get(pageNumber));
+                } else {
+                    classificationPage.getImages().addAll(signatures.get(pageNumber));
+                }
+            }
+
            tableExtractionService.extractTables(cleanRulings, classificationPage);

            buildPageStatistics(classificationPage);
@@ -313,19 +308,9 @@ public class LayoutParsingPipeline {
            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }

-        List<ClassificationSection> sections = new ArrayList<>();
-        for (var page : classificationPages) {
-            page.getTextBlocks().forEach(block -> {
-                block.setPage(page.getPageNumber());
-                var section = sectionsBuilderService.buildTextBlock(List.of(block), "a");
-                sections.add(section);
-            });
-        }
-        classificationDocument.setSections(sections);
-
        log.info("Building Sections for {}", identifier);
-//        sectionsBuilderService.buildSections(classificationDocument);
-//        sectionsBuilderService.addImagesToSections(classificationDocument);
+        sectionsBuilderService.buildSections(classificationDocument);
+        sectionsBuilderService.addImagesToSections(classificationDocument);
        return classificationDocument;
    }

@@ -359,9 +344,7 @@ public class LayoutParsingPipeline {

    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {

-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-        }
+        document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
@@ -10,8 +10,6 @@ import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.Optional;

-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -22,6 +20,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.Researc
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

 import io.micrometer.observation.annotation.Observed;
@@ -79,6 +78,14 @@ public class LayoutParsingStorageService {
        }
    }

+    public VisualLayoutParsingResponse getVisualLayoutParsingFile(String storageId) throws IOException {
+
+        try (InputStream inputStream = getObject(storageId)) {
+            VisualLayoutParsingResponse visualLayoutParsingResponse = objectMapper.readValue(inputStream, VisualLayoutParsingResponse.class);
+            return visualLayoutParsingResponse;
+        }
+    }
+

    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {
@@ -12,16 +12,15 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRul
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;

 import lombok.Data;
-import lombok.Getter;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;

@Data
@RequiredArgsConstructor
 public class ClassificationPage {

    @NonNull
-    @Getter
    private List<AbstractPageBlock> textBlocks;

    private List<ClassifiedImage> images = new ArrayList<>();
@@ -10,6 +10,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -30,6 +31,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Document implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
    Set<Page> pages;
    DocumentTree documentTree;
    Integer numberOfPages;
@@ -56,13 +60,15 @@ public class Document implements GenericSemanticNode {

    public List<Section> getMainSections() {

-        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node).collect(Collectors.toList());
+        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node)
+                .collect(Collectors.toList());
    }


    public Stream<TextBlock> streamTerminalTextBlocksInOrder() {

-        return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock);
+        return streamAllNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock);
    }


@@ -83,13 +89,16 @@ public class Document implements GenericSemanticNode {
    @Override
    public Headline getHeadline() {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build());
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node)
+                .findFirst()
+                .orElse(Headline.builder().build());
    }


    private Stream<SemanticNode> streamAllNodes() {

-        return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode);
+        return documentTree.allEntriesInOrder()
+                .map(DocumentTree.Entry::getNode);
    }


@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -26,6 +27,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Footer implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
    List<Integer> treeId;
    TextBlock leafTextBlock;

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Header implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;
    TextBlock leafTextBlock;

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Headline implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;
    TextBlock leafTextBlock;

@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -29,6 +30,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Image implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
    List<Integer> treeId;
    String id;

@@ -66,7 +70,9 @@ public class Image implements GenericSemanticNode {
    @Override
    public TextBlock getTextBlock() {

-        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
    }


@@ -6,6 +6,8 @@ public enum ImageType {
    LOGO,
    FORMULA,
    SIGNATURE,
+
+    SIGNATURE_VISUAL,
    OTHER,
    OCR;

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -24,6 +25,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Paragraph implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
    List<Integer> treeId;
    TextBlock leafTextBlock;

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -27,6 +28,8 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Section implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;

    TextBlock textBlock;
@@ -50,7 +53,8 @@ public class Section implements GenericSemanticNode {

    public boolean hasTables() {

-        return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent();
+        return streamAllSubNodesOfType(NodeType.TABLE).findAny()
+                .isPresent();
    }


@@ -58,7 +62,9 @@ public class Section implements GenericSemanticNode {
    public TextBlock getTextBlock() {

        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
+                    .map(SemanticNode::getLeafTextBlock)
+                    .collect(new TextBlockCollector());
        }
        return textBlock;
    }
@@ -12,6 +12,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
@@ -335,6 +336,23 @@ public interface SemanticNode {
    }


+    /**
+     * returns the set of layoutengines.
+     *
+     * @return set of layoutengines.
+     */
+    Set<LayoutEngine> getEngines();
+
+
+    /**
+     * adds a layoutengine to the set.
+     */
+    default void addEngine(LayoutEngine engine) {
+
+        getEngines().add(engine);
+    }
+
+
    /**
     * Streams all children located directly underneath this node in the DocumentTree.
     *
@@ -426,6 +444,7 @@ public interface SemanticNode {
    /**
     * TODO: this produces unwanted results for sections spanning multiple columns.
     * Computes the Union of the bounding boxes of all children recursively.
+     *
     * @return The union of the BoundingBoxes of all children
     */
    private Map<Page, Rectangle2D> getBBoxFromChildren() {
@@ -12,6 +12,7 @@ import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -31,6 +32,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Table implements SemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;
    DocumentTree documentTree;

@@ -45,6 +48,7 @@ public class Table implements SemanticNode {
    @EqualsAndHashCode.Exclude
    Map<Page, Rectangle2D> bBoxCache;

+
    /**
     * Streams all entities in this table, that appear in a row, which contains any of the provided strings.
     *
@@ -53,8 +57,7 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {

-        return IntStream.range(0, numberOfRows)
-                .boxed()
+        return IntStream.range(0, numberOfRows).boxed()
                .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                .flatMap(this::streamRow)
                .map(TableCell::getEntities)
@@ -71,8 +74,11 @@ public class Table implements SemanticNode {
     */
    public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {

-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
-        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
+        String rowText = streamRow(row).map(TableCell::getTextBlock)
+                .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
+        return strings.stream()
+                .map(String::toLowerCase)
+                .allMatch(rowText::contains);
    }


@@ -85,9 +91,13 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {

-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
        return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
-                .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -100,9 +110,13 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {

-        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
        return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -114,12 +128,15 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {

-        List<Integer> rowsWithEntityOfType = IntStream.range(0, numberOfRows)
-                .boxed()
-                .filter(rowNumber -> streamEntityTypesInRow(rowNumber).anyMatch(existingType -> types.stream().anyMatch(typeToCheck -> typeToCheck.equals(existingType))))
+        List<Integer> rowsWithEntityOfType = IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamEntityTypesInRow(rowNumber).anyMatch(existingType -> types.stream()
+                        .anyMatch(typeToCheck -> typeToCheck.equals(existingType))))
                .toList();

-        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
+        return rowsWithEntityOfType.stream()
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -131,18 +148,24 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {

-        List<Integer> rowsWithNoEntityOfType = IntStream.range(0, numberOfRows)
-                .boxed()
-                .filter(rowNumber -> streamEntityTypesInRow(rowNumber).noneMatch(existingType -> types.stream().anyMatch(typeToCheck -> typeToCheck.equals(existingType))))
+        List<Integer> rowsWithNoEntityOfType = IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamEntityTypesInRow(rowNumber).noneMatch(existingType -> types.stream()
+                        .anyMatch(typeToCheck -> typeToCheck.equals(existingType))))
                .toList();

-        return rowsWithNoEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
+        return rowsWithNoEntityOfType.stream()
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


    private Stream<String> streamEntityTypesInRow(Integer rowNumber) {

-        return streamRow(rowNumber).map(TableCell::getEntities).flatMap(Collection::stream).map(RedactionEntity::getType).distinct();
+        return streamRow(rowNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .map(RedactionEntity::getType)
+                .distinct();
    }


@@ -159,7 +182,8 @@ public class Table implements SemanticNode {
            throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
        }
        int idx = row * numberOfCols + col;
-        return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
+        return (TableCell) documentTree.getEntryById(treeId).getChildren()
+                .get(idx).getNode();
    }


@@ -196,7 +220,8 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamCol(int col) {

-        return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
+        return IntStream.range(0, numberOfRows).boxed()
+                .map(row -> getCell(row, col));
    }


@@ -208,9 +233,11 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamRow(int row) {

-        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
+        return IntStream.range(0, numberOfCols).boxed()
+                .map(col -> getCell(row, col));
    }

+
    /**
     * Streams all TableCells row-wise and filters them with header == true.
     *
@@ -231,7 +258,8 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamHeadersForCell(int row, int col) {

-        return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
+        return Stream.concat(streamRow(row), streamCol(col))
+                .filter(TableCell::isHeader);
    }


@@ -304,7 +332,9 @@ public class Table implements SemanticNode {
    public TextBlock getTextBlock() {

        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
+                    .map(SemanticNode::getLeafTextBlock)
+                    .collect(new TextBlockCollector());
        }
        return textBlock;
    }
@@ -315,6 +345,8 @@ public class Table implements SemanticNode {

        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
    }
+
+
    @Override
    public Map<Page, Rectangle2D> getBBox() {

@@ -323,4 +355,5 @@ public class Table implements SemanticNode {
        }
        return bBoxCache;
    }
+
 }
@@ -7,6 +7,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class TableCell implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;
    int row;
    int col;
@@ -4,18 +4,21 @@ import java.awt.geom.Rectangle2D;

 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;

+import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;

@Data
@RequiredArgsConstructor
+@AllArgsConstructor
 public class ClassifiedImage {

    @NonNull
    private Rectangle2D position;
    @NonNull
    private ImageType imageType;
+    private boolean sourceByAi;
    private boolean isAppendedToSection;
    @NonNull
    private boolean hasTransparency;
@@ -76,4 +76,14 @@ public class Cell extends Rectangle {
        return this.getHeight() >= MIN_SIZE && this.getWidth() >= MIN_SIZE;
    }

+    public boolean nearlyIntersects(Cell other) {
+
+        if (this.getHeight() <= 0 || other.getHeight() <= 0) {
+            return false;
+        }
+        double x0 = this.getX() + 2;
+        double y0 = this.getY() + 2;
+        return (other.x + other.width > x0 && other.y + other.height > y0 && other.x < x0 + this.getWidth() - 2 && other.y < y0 + this.getHeight() - 2);
+    }
+
 }
@@ -1,14 +1,12 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -21,7 +19,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
 public class TablePageBlock extends AbstractPageBlock {

-    private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();
+    private final TreeMap<CellPosition, Cell> cellTreeMap = new TreeMap<>();

    private final int rotation;
    @Getter
@@ -30,10 +28,14 @@ public class TablePageBlock extends AbstractPageBlock {
    private int unrotatedRowCount;
    private int unrotatedColCount;
    private List<List<Cell>> rows;
+    @Getter
+    @Setter
+    private List<Cell> cells;


    public TablePageBlock(List<Cell> cells, Rectangle area, int rotation) {

+        this.cells = cells;
        addCells(cells);
        minX = area.getLeft();
        minY = area.getBottom();
@@ -50,6 +52,7 @@ public class TablePageBlock extends AbstractPageBlock {
        return getColCount() == 0 || getRowCount() == 0;
    }

+
    public List<List<Cell>> getRows() {

        if (rows == null) {
@@ -80,7 +83,10 @@ public class TablePageBlock extends AbstractPageBlock {

    public int getColCount() {

-        return getRows().stream().mapToInt(List::size).max().orElse(0);
+        return getRows().stream()
+                .mapToInt(List::size)
+                .max()
+                .orElse(0);

    }

@@ -120,7 +126,8 @@ public class TablePageBlock extends AbstractPageBlock {
                List<Cell> cellsToTheTop = new ArrayList<>();
                for (int i = 0; i < rowIndex; i++) {
                    try {
-                        cellsToTheTop.add(rows.get(i).get(colIndex));
+                        cellsToTheTop.add(rows.get(i)
+                                                  .get(colIndex));
                    } catch (IndexOutOfBoundsException e) {
                        log.debug("No cell {} in row {}, ignoring.", colIndex, rowIndex);
                    }
@@ -135,7 +142,8 @@ public class TablePageBlock extends AbstractPageBlock {
                if (lastHeaderCell != null) {
                    cell.getHeaderCells().add(lastHeaderCell);
                }
-                if (!cell.getTextBlocks().isEmpty() && cell.getTextBlocks().get(0).getMostPopularWordStyle().equals("bold")) {
+                if (!cell.getTextBlocks().isEmpty() && cell.getTextBlocks()
+                        .get(0).getMostPopularWordStyle().equals("bold")) {
                    cell.setHeaderCell(true);
                }
            }
@@ -151,7 +159,7 @@ public class TablePageBlock extends AbstractPageBlock {
            for (int i = 0; i < unrotatedColCount; i++) { // rows
                List<Cell> lastRow = new ArrayList<>();
                for (int j = unrotatedRowCount - 1; j >= 0; j--) { // cols
-                    Cell cell = cells.get(new CellPosition(j, i));
+                    Cell cell = cellTreeMap.get(new CellPosition(j, i));
                    if (cell != null) {
                        lastRow.add(cell);
                    }
@@ -162,7 +170,7 @@ public class TablePageBlock extends AbstractPageBlock {
            for (int i = unrotatedColCount - 1; i >= 0; i--) { // rows
                List<Cell> lastRow = new ArrayList<>();
                for (int j = 0; j < unrotatedRowCount; j++) { // cols
-                    Cell cell = cells.get(new CellPosition(j, i));
+                    Cell cell = cellTreeMap.get(new CellPosition(j, i));
                    if (cell != null) {
                        lastRow.add(cell);
                    }
@@ -173,7 +181,7 @@ public class TablePageBlock extends AbstractPageBlock {
            for (int i = 0; i < unrotatedRowCount; i++) {
                List<Cell> lastRow = new ArrayList<>();
                for (int j = 0; j < unrotatedColCount; j++) {
-                    Cell cell = cells.get(new CellPosition(i, j)); // JAVA_8 use getOrDefault()
+                    Cell cell = cellTreeMap.get(new CellPosition(i, j)); // JAVA_8 use getOrDefault()
                    if (cell != null) {
                        lastRow.add(cell);
                    }
@@ -187,17 +195,6 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-    private void add(Cell chunk, int row, int col) {
-
-        unrotatedRowCount = Math.max(unrotatedRowCount, row + 1);
-        unrotatedColCount = Math.max(unrotatedColCount, col + 1);
-
-        CellPosition cp = new CellPosition(row, col);
-        cells.put(cp, chunk);
-
-    }
-
-
    private void addCells(List<Cell> cells) {

        if (cells.isEmpty()) {
@@ -206,11 +203,12 @@ public class TablePageBlock extends AbstractPageBlock {

        cells.removeIf(cell -> cell.getWidth() < 1.1 || cell.getHeight() < 1.1);

-        List<List<Cell>> rowsOfCells = calculateStructure(cells);
+        List<List<Cell>> rowsOfCellsMatrix = calculateTableStructure(cells);

-        for (int i = 0; i < rowsOfCells.size(); i++) {
-            for (int j = 0; j < rowsOfCells.get(i).size(); j++) {
-                add(rowsOfCells.get(i).get(j), i, j);
+        for (int i = 0; i < rowsOfCellsMatrix.size(); i++) {
+            for (int j = 0; j < rowsOfCellsMatrix.get(i).size(); j++) {
+                addCellToRowAndCol(rowsOfCellsMatrix.get(i)
+                                           .get(j), i, j);
            }
        }

@@ -221,29 +219,36 @@ public class TablePageBlock extends AbstractPageBlock {
     * Calculates the structure of the table. For spanning rows and columns multiple cells with the same values will be inserted.
     *
     * @param cells The found cells
-     * @return TablePageBlock Structure
+     * @return TablePageBlock Structure as a rows of cells matrix
     */
-    private List<List<Cell>> calculateStructure(List<Cell> cells) {
-
-        List<List<Cell>> matrix = new ArrayList<>();
+    private List<List<Cell>> calculateTableStructure(List<Cell> cells) {

        if (cells.isEmpty()) {
-            return matrix;
+            return new ArrayList<>();
        }

        Set<Float> uniqueX = new HashSet<>();
        Set<Float> uniqueY = new HashSet<>();
-        cells.stream().filter(c -> !c.getTextBlocks().isEmpty() || c.getHeight() > 3 && c.getWidth() > 3).forEach(c -> {
-            uniqueX.add(c.getLeft());
-            uniqueX.add(c.getRight());
-            uniqueY.add(c.getBottom());
-            uniqueY.add(c.getTop());
-        });
+        cells.stream()
+                .filter(c -> !c.getTextBlocks().isEmpty() || c.getHeight() > 3 && c.getWidth() > 3)
+                .forEach(c -> {
+                    uniqueX.add(c.getLeft());
+                    uniqueX.add(c.getRight());
+                    uniqueY.add(c.getBottom());
+                    uniqueY.add(c.getTop());
+                });

-        var sortedUniqueX = uniqueX.stream().sorted().toList();
-        var sortedUniqueY = uniqueY.stream().sorted().toList();
+        var sortedUniqueX = uniqueX.stream()
+                .sorted()
+                .toList();
+        var sortedUniqueY = uniqueY.stream()
+                .sorted()
+                .toList();
+
+        List<List<Cell>> rowsOfCells = new ArrayList<>();

        Float prevY = null;
+
        for (Float y : sortedUniqueY) {

            List<Cell> row = new ArrayList<>();
@@ -254,42 +259,81 @@ public class TablePageBlock extends AbstractPageBlock {
                if (prevY != null && prevX != null) {
                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));

-                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
-
-                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
                    if (cell.hasMinimumSize()) {
+
+                        cells.stream()
+                                .filter(cell::nearlyIntersects)
+                                .forEach(intersectingCell -> cell.getTextBlocks().addAll(intersectingCell.getTextBlocks()));
+
                        row.add(cell);
                    }
                }
                prevX = x;
            }

-            if (prevY != null && prevX != null && !row.isEmpty()) {
-                matrix.add(row);
+            // exclude empty rows and rows where all text blocks are empty
+            if (prevY != null && prevX != null && !row.isEmpty() && !row.stream()
+                    .allMatch(cell -> cell.getTextBlocks().isEmpty())) {
+
+                rowsOfCells.add(row);
            }
            prevY = y;
        }

-        Collections.reverse(matrix);
+        Collections.reverse(rowsOfCells);

-        return matrix;
-    }
-
-
-
-    public boolean intersects(Cell cell1, Cell cell2) {
-        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
-            return false;
+        // now cells are removed which are part of a column without any text blocks
+        // this is done by first computing the inverse matrix which contains call columns of cells
+        // then the column indices that have to be removed are determined
+        List<List<Cell>> columnsOfCells = new ArrayList<>();
+        int maxRowLength = rowsOfCells.stream()
+                .map(List::size)
+                .max(java.util.Comparator.naturalOrder())
+                .orElse(0);
+        for (int i = 0; i < maxRowLength; i++) {
+            columnsOfCells.add(new ArrayList<>());
        }
-        double x0 = cell1.getX() + 2;
-        double y0 = cell1.getY() + 2;
-        return (cell2.x + cell2.width > x0 &&
-                cell2.y + cell2.height > y0 &&
-                cell2.x < x0 + cell1.getWidth() -2 &&
-                cell2.y < y0 + cell1.getHeight() -2);
+
+        for (List<Cell> row : rowsOfCells) {
+            for (int j = 0; j < row.size(); j++) {
+                columnsOfCells.get(j).add(row.get(j));
+            }
+        }
+
+        List<Integer> columnIndicesToRemove = new ArrayList<>();
+        int columnIndex = 0;
+        for (List<Cell> col : columnsOfCells) {
+            if (col.stream()
+                    .allMatch(cell -> cell.getTextBlocks().isEmpty())) {
+                columnIndicesToRemove.add(columnIndex);
+            }
+            columnIndex++;
+        }
+        columnIndicesToRemove.sort(Collections.reverseOrder());
+
+        // update all rows so that the values of the empty columns get removed
+        var rowsOfCellsBefore = new ArrayList<>(rowsOfCells);
+        rowsOfCells = new ArrayList<>();
+        for (List<Cell> row : rowsOfCellsBefore) {
+            var updatedRow = new ArrayList<>(row);
+            columnIndicesToRemove.forEach(idxToRemove -> updatedRow.remove(updatedRow.get(idxToRemove)));
+            rowsOfCells.add(updatedRow);
+        }
+
+        return rowsOfCells;
    }


+    private void addCellToRowAndCol(Cell cell, int row, int col) {
+
+        unrotatedRowCount = Math.max(unrotatedRowCount, row + 1);
+        unrotatedColCount = Math.max(unrotatedColCount, col + 1);
+
+        CellPosition cp = new CellPosition(row, col);
+        cellTreeMap.put(cp, cell);
+
+    }
+

    @Override
    public String getText() {
@@ -314,7 +358,7 @@ public class TablePageBlock extends AbstractPageBlock {
                            if (!first) {
                                sb.append("\n");
                            }
-                            sb.append('\"').append(textBlock.getText().replaceAll("\"", "\\\"")).append('\"');
+                            sb.append('\"').append(textBlock.getText().replaceAll("\"", "\"")).append('\"');
                            first = false;
                        }
                    }
@@ -328,8 +372,6 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-
-
    public String getTextAsHtml() {

        StringBuilder sb = new StringBuilder();
@@ -45,9 +45,6 @@ public class RedTextPosition {
    @JsonIgnore
    private String fontName;

-    @JsonIgnore
-    private RedTextPosition parent;
-

    @SneakyThrows
    public static RedTextPosition fromTextPosition(TextPosition textPosition) {
@@ -17,7 +17,6 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.Getter;
 import lombok.NoArgsConstructor;

@EqualsAndHashCode(callSuper = true)
@@ -28,7 +27,6 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {

    @Builder.Default
-    @Getter
    private List<TextPositionSequence> sequences = new ArrayList<>();

    @JsonIgnore
@@ -75,7 +73,7 @@ public class TextPageBlock extends AbstractPageBlock {

        return sequences.get(0).getPageWidth();
    }
-
+    

    public static TextPageBlock merge(List<TextPageBlock> textBlocksToMerge) {

@@ -84,7 +82,6 @@ public class TextPageBlock extends AbstractPageBlock {
        return fromTextPositionSequences(sequences);
    }

-
    public static TextPageBlock fromTextPositionSequences(List<TextPositionSequence> wordBlockList) {

        TextPageBlock textBlock = null;
@@ -136,6 +133,7 @@ public class TextPageBlock extends AbstractPageBlock {
    }


+
    /**
     * Returns the minX value in pdf coordinate system.
     * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -55,18 +55,6 @@ public class TextPositionSequence implements CharSequence {
    }


-    public TextPositionSequence(List<RedTextPosition> textPositions, int page) {
-
-        this.textPositions = textPositions;
-        this.page = page;
-        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-        this.isParagraphStart = false;
-    }
-
-
    @Override
    public int length() {

@@ -0,0 +1,92 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingBox;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class VisualLayoutParsingAdapter {
+
+    private static String SIGNATURES = "signature";
+
+
+    public Map<Integer, List<VisualLayoutParsingResult>> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
+
+        Map<Integer, List<VisualLayoutParsingResult>> tableCells = new HashMap<>();
+        visualLayoutParsingResponse.getData()
+                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertTableCells(tableData.getBoxes())));
+
+        return tableCells;
+
+    }
+
+
+    public Map<Integer, List<ClassifiedImage>> buildExtractedSignaturesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
+
+        Map<Integer, List<ClassifiedImage>> signatures = new HashMap<>();
+        if (visualLayoutParsingResponse.getData() != null) {
+            visualLayoutParsingResponse.getData()
+                    .forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx() + 1, tableCell -> new ArrayList<>())
+                            .addAll(convertSignatures(tableData.getPage_idx(), tableData.getBoxes())));
+        }
+        return signatures;
+
+    }
+
+
+    public List<VisualLayoutParsingResult> convertTableCells(List<VisualLayoutParsingBox> tableObjects) {
+
+        List<VisualLayoutParsingResult> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            VisualLayoutParsingResult result = new VisualLayoutParsingResult();
+            result.setX0(t.getBox().getX1());
+            result.setX1(t.getBox().getX2());
+            result.setY0(t.getBox().getY1());
+            result.setY1(t.getBox().getY2());
+            result.setWidth(result.getX1() - result.getX0());
+            result.setHeight(result.getY1() - result.getY0());
+            result.setLabel(t.getLabel());
+            parsedTableCells.add(result);
+        });
+
+        return parsedTableCells;
+
+    }
+
+
+    public List<ClassifiedImage> convertSignatures(int pageNumber, List<VisualLayoutParsingBox> tableObjects) {
+
+        List<ClassifiedImage> signatures = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            if (t.getLabel().equals(SIGNATURES)) {
+                ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(),
+                        t.getBox().getY1(),
+                        t.getBox().getX2() - t.getBox().getX1(),
+                        t.getBox().getY2() - t.getBox().getY1()), ImageType.SIGNATURE, true, false, false, pageNumber);
+
+                signatures.add(signature);
+            }
+        });
+
+        return signatures;
+
+    }
+
+}
@@ -0,0 +1,18 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingBox {
+
+    private VisualLayoutParsingBoxValue box;
+    private String label;
+    private float probability;
+
+}
@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingBoxValue {
+
+    private float x1;
+    private float y1;
+    private float x2;
+    private float y2;
+
+}
@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingData {
+
+    private int page_idx;
+
+    private List<VisualLayoutParsingBox> boxes;
+
+}
@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingResponse {
+
+    private String dossierId;
+    private String fileId;
+    private String targetFileExtension;
+    private String responseFileExtension;
+    private String X_TENANT_ID;
+    private List<VisualLayoutParsingData> data;
+
+}
@@ -0,0 +1,22 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingResult {
+
+    private float x0;
+    private float y0;
+    private float x1;
+    private float y1;
+    private float width;
+    private float height;
+    private String label;
+
+}
@@ -25,7 +25,8 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RulingCleaningService {

-    private static final float THRESHOLD = 6;
+    private static final float THRESHOLD_Y = 6;
+    private static final float THRESHOLD_X = 2;


    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {
@@ -81,7 +82,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
+            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD_X) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -108,7 +109,7 @@ public class RulingCleaningService {

        for (Point2D p : points.subList(1, points.size() - 1)) {
            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
+            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD_Y) {
                groupedPoints.get(groupedPoints.size() - 1).add(p);
            } else {
                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -240,7 +240,7 @@ public class SectionsBuilderService {
    }


-    public ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
+    private ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {

        ClassificationSection section = new ClassificationSection();

@@ -5,7 +5,6 @@ import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -25,55 +24,62 @@ import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparis
@Service
 public class TableExtractionService {

-    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (arg0, arg1) -> {
+    private static final int MAX_TABLE_OUTER_POINT_TOLERANCE = 10;
+    private static final int MAX_TABLE_CONTAINED_CELLS_WITH_TEXT = 1;
+    private static final float SPREADSHEET_AREA_TOLERANCE = 0.001f;
+
+    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (point1, point2) -> {

        int rv = 0;
-        float arg0X = DoubleComparisons.round(arg0.getX(), 2);
-        float arg0Y = DoubleComparisons.round(arg0.getY(), 2);
-        float arg1X = DoubleComparisons.round(arg1.getX(), 2);
-        float arg1Y = DoubleComparisons.round(arg1.getY(), 2);
+        float point1X = DoubleComparisons.round(point1.getX(), 2);
+        float point1Y = DoubleComparisons.round(point1.getY(), 2);
+        float point2X = DoubleComparisons.round(point2.getX(), 2);
+        float point2Y = DoubleComparisons.round(point2.getY(), 2);

-        if (arg0X > arg1X) {
+        if (point1X > point2X) {
            rv = 1;
-        } else if (arg0X < arg1X) {
+        } else if (point1X < point2X) {
            rv = -1;
-        } else if (arg0Y > arg1Y) {
+        } else if (point1Y > point2Y) {
            rv = 1;
-        } else if (arg0Y < arg1Y) {
+        } else if (point1Y < point2Y) {
            rv = -1;
        }
        return rv;
    };
-    private static final Comparator<Point2D> POINT_COMPARATOR = (arg0, arg1) -> {
+    private static final Comparator<Point2D> Y_FIRST_POINT_COMPARATOR = (point1, point2) -> {

        int rv = 0;
-        float arg0X = DoubleComparisons.round(arg0.getX(), 2);
-        float arg0Y = DoubleComparisons.round(arg0.getY(), 2);
-        float arg1X = DoubleComparisons.round(arg1.getX(), 2);
-        float arg1Y = DoubleComparisons.round(arg1.getY(), 2);
+        float point1X = DoubleComparisons.round(point1.getX(), 2);
+        float point1Y = DoubleComparisons.round(point1.getY(), 2);
+        float point2X = DoubleComparisons.round(point2.getX(), 2);
+        float point2Y = DoubleComparisons.round(point2.getY(), 2);

-        if (arg0Y > arg1Y) {
+        if (point1Y > point2Y) {
            rv = 1;
-        } else if (arg0Y < arg1Y) {
+        } else if (point1Y < point2Y) {
            rv = -1;
-        } else if (arg0X > arg1X) {
+        } else if (point1X > point2X) {
            rv = 1;
-        } else if (arg0X < arg1X) {
+        } else if (point1X < point2X) {
            rv = -1;
        }
        return rv;
    };

+    private static final Comparator<Cell> CELL_SIZE_COMPARATOR = (cell1, cell2) -> {

-    public boolean contains(Cell cell, double x, double y, double w, double h) {
+        Double cell1Size = cell1.getHeight() * cell1.getWidth();
+        Double cell2Size = cell2.getHeight() * cell2.getWidth();
+        return cell1Size.compareTo(cell2Size);
+    };

-        if (cell.isEmpty() || w <= 0 || h <= 0) {
-            return false;
-        }
-        double x0 = cell.getX();
-        double y0 = cell.getY();
-        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
-    }
+    private static final Comparator<Rectangle> RECTANGLE_SIZE_COMPARATOR = (rect1, rect2) -> {
+
+        Double rect1Size = rect1.getHeight() * rect1.getWidth();
+        Double rect2Size = rect2.getHeight() * rect2.getWidth();
+        return rect1Size.compareTo(rect2Size);
+    };


    /**
@@ -89,22 +95,18 @@ public class TableExtractionService {
     * @param cleanRulings The lines used to build the table.
     * @param page         Page object that contains textblocks and statistics.
     */
+
    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {

        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-        List<TextPageBlock> toBeRemoved = new ArrayList<>();
+        // sort cells by size (height * width) ascending so that textBlocks are always assigned to the smallest cells that contain them
+        cells.sort(CELL_SIZE_COMPARATOR);

        for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
            TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
            for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && contains(cell,
-                        textBlock.getPdfMinX(),
-                        textBlock.getPdfMinY(),
-                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
-                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
+                if (cell.hasMinimumSize() && doesCellContainTextBlock(cell, textBlock)) {
                    cell.addTextBlock(textBlock);
-                    toBeRemoved.add(textBlock);
                    break;
                }
            }
@@ -114,39 +116,70 @@ public class TableExtractionService {
        DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);

        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);
+        // sort spreadsheetAreas by size (height * width) ascending so that cells are placed in the smallest tables first
+        // this way no cell duplication occurs when tables are contained in other tables and only the most inner table contains the cells
+        spreadsheetAreas.sort(RECTANGLE_SIZE_COMPARATOR);

        List<TablePageBlock> tables = new ArrayList<>();
        for (Rectangle area : spreadsheetAreas) {

-            List<Cell> overlappingCells = new ArrayList<>();
+            List<Cell> containedCells = new ArrayList<>();
            for (Cell c : cells) {
-                if (c.hasMinimumSize() && c.intersects(area)) {
-                    overlappingCells.add(c);
+                if (c.hasMinimumSize() && area.contains(c)) {
+                    containedCells.add(c);
                }
            }
-            tables.add(new TablePageBlock(overlappingCells, area, page.getRotation()));
+
+            var containedCellsWithText = containedCells.stream()
+                    .filter(cell -> !cell.getTextBlocks().isEmpty())
+                    .count();
+
+            // verify if table would contain fewer cells with text than the threshold allows
+            if (containedCellsWithText >= MAX_TABLE_CONTAINED_CELLS_WITH_TEXT) {
+                tables.add(new TablePageBlock(containedCells, area, page.getRotation()));
+                cells.removeAll(containedCells);
+            }
        }

        for (TablePageBlock table : tables) {
            int position = -1;

-            Iterator<AbstractPageBlock> itty = page.getTextBlocks().iterator();
-            while (itty.hasNext()) {
-                AbstractPageBlock textBlock = itty.next();
-                if (textBlock instanceof TextPageBlock ? table.containsBlock((TextPageBlock) textBlock) : table.contains(textBlock) && position == -1) {
-                    position = page.getTextBlocks().indexOf(textBlock);
+            for (AbstractPageBlock pageBlock : page.getTextBlocks()) {
+                if (pageBlock instanceof TextPageBlock ? table.containsBlock((TextPageBlock) pageBlock) : table.contains(pageBlock) && position == -1) {
+                    position = page.getTextBlocks().indexOf(pageBlock);
                }
            }
            if (position != -1) {
                page.getTextBlocks().add(position, table);
+
+                var toBeRemoved = table.getCells()
+                        .stream()
+                        .map(Cell::getTextBlocks)
+                        .flatMap(List::stream)
+                        .toList();
+                // remove text blocks from the page that were also added with the table (from its contained cells)
+                page.getTextBlocks().removeAll(toBeRemoved);
            }
        }
-
-        page.getTextBlocks().removeAll(toBeRemoved);
    }


-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    private boolean doesCellContainTextBlock(Cell cell, TextPageBlock textBlock) {
+
+        double x = textBlock.getPdfMinX();
+        double y = textBlock.getPdfMinY();
+        double w = textBlock.getPdfMaxX() - textBlock.getPdfMinX();
+        double h = textBlock.getPdfMaxY() - textBlock.getPdfMinY();
+        if (cell.isEmpty() || w <= 0 || h <= 0) {
+            return false;
+        }
+        double x0 = cell.getX();
+        double y0 = cell.getY();
+        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
+    }
+
+
+    private List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

        // Fix for 211.pdf
        for (Ruling r : horizontalRulingLines) {
@@ -160,7 +193,7 @@ public class TableExtractionService {
        List<Cell> cellsFound = new ArrayList<>();
        Map<Point2D, Ruling[]> intersectionPoints = Ruling.findIntersections(horizontalRulingLines, verticalRulingLines);
        List<Point2D> intersectionPointsList = new ArrayList<>(intersectionPoints.keySet());
-        intersectionPointsList.sort(POINT_COMPARATOR);
+        intersectionPointsList.sort(Y_FIRST_POINT_COMPARATOR);

        for (int i = 0; i < intersectionPointsList.size(); i++) {
            Point2D topLeft = intersectionPointsList.get(i);
@@ -186,13 +219,14 @@ public class TableExtractionService {
                    continue;
                }
                for (Point2D yPoint : yPoints) {
-                    // is there an horizontal edge b/w topLeft and yPoint ?
+                    // is there a horizontal edge b/w topLeft and yPoint ?
                    if (!hv[0].equals(intersectionPoints.get(yPoint)[0])) {
                        continue;
                    }
                    Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY());
-                    if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(
-                            intersectionPoints.get(yPoint)[1])) {
+                    if (intersectionPoints.containsKey(btmRight)
+                        && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0])
+                        && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
                        cellsFound.add(new Cell(topLeft, btmRight));
                        break outer;
                    }
@@ -214,7 +248,6 @@ public class TableExtractionService {
        Set<Point2D> pointSet = new HashSet<>();
        Map<Point2D, Point2D> edgesH = new HashMap<>();
        Map<Point2D, Point2D> edgesV = new HashMap<>();
-        int i = 0;

        for (Rectangle cell : cells) {
            for (Point2D pt : cell.getPoints()) {
@@ -231,8 +264,9 @@ public class TableExtractionService {
        pointsSortX.sort(X_FIRST_POINT_COMPARATOR);
        // Y first sort
        List<Point2D> pointsSortY = new ArrayList<>(pointSet);
-        pointsSortY.sort(POINT_COMPARATOR);
+        pointsSortY.sort(Y_FIRST_POINT_COMPARATOR);

+        int i = 0;
        while (i < pointSet.size()) {
            float currY = (float) pointsSortY.get(i).getY();
            while (i < pointSet.size() && DoubleComparisons.feq(pointsSortY.get(i).getY(), currY)) {
@@ -257,7 +291,8 @@ public class TableExtractionService {
        Point2D nextVertex;
        while (!edgesH.isEmpty()) {
            ArrayList<PolygonVertex> polygon = new ArrayList<>();
-            Point2D first = edgesH.keySet().iterator().next();
+            Point2D first = edgesH.keySet()
+                    .iterator().next();
            polygon.add(new PolygonVertex(first, Direction.HORIZONTAL));
            edgesH.remove(first);

@@ -301,7 +336,14 @@ public class TableExtractionService {
                bottom = (float) Math.max(bottom, pt.point.getY());
                right = (float) Math.max(right, pt.point.getX());
            }
-            rectangles.add(new Rectangle(top, left, right - left, bottom - top));
+
+            // do not add polygons with too many outer points as they are unlikely to be tables
+            if (poly.size() <= MAX_TABLE_OUTER_POINT_TOLERANCE) {
+                rectangles.add(new Rectangle(top - SPREADSHEET_AREA_TOLERANCE,
+                                             left - SPREADSHEET_AREA_TOLERANCE,
+                                             right - left + 2 * SPREADSHEET_AREA_TOLERANCE,
+                                             bottom - top + 2 * SPREADSHEET_AREA_TOLERANCE));
+            }
        }

        return rectangles;
@@ -57,7 +57,7 @@ public class RedactManagerBlockificationService {
            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());

-            if (prev != null && (splitByDir || isSplitByRuling)) {
+            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {

                Orientation prevOrientation = null;
                if (!chunkBlockList.isEmpty()) {
@@ -167,7 +167,7 @@ public class RedactManagerBlockificationService {
    }


-    public TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    private TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {

        TextPageBlock textBlock = null;

@@ -1,153 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.LineBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.NearestNeighbourService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ReadingOrderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.SpacingService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.ZoneBuilderService;
-
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class DocstrumSegmentationService {
-
-    private static final double MAX_VERTICAL_MERGE_DIST = 0.5;
-
-    private final NearestNeighbourService nearestNeighbourService;
-    private final SpacingService spacingService;
-    private final LineBuilderService lineBuilderService;
-    private final ZoneBuilderService zoneBuilderService;
-    private final ReadingOrderService readingOrderService;
-
-
-    public List<Zone> segmentPage(List<TextPositionSequence> textPositions) {
-
-        var positions = textPositions.stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
-
-        var characters = positions.stream().map(Character::new).collect(Collectors.toList());
-
-        nearestNeighbourService.findNearestNeighbors(characters);
-
-        var characterSpacing = spacingService.computeCharacterSpacing(characters);
-        var lineSpacing = spacingService.computeLineSpacing(characters);
-
-        var lines = lineBuilderService.buildLines(characters, characterSpacing, lineSpacing);
-
-        var zones = zoneBuilderService.buildZones(lines, characterSpacing, lineSpacing);
-
-        zones = mergeLinesInZones(zones, characterSpacing, Double.NEGATIVE_INFINITY, 0.0, 0.0, lineSpacing * MAX_VERTICAL_MERGE_DIST);
-
-        return readingOrderService.resolve(zones, false);
-    }
-
-//    private List<Zone> mergeZones(List<Zone> zones, double tolerance) {
-//
-//        List<BxBounds> bounds = new ArrayList<BxBounds>(zones.size());
-//        for (List<ComponentLine> zone : zones) {
-//            BxBoundsBuilder builder = new BxBoundsBuilder();
-//            for (ComponentLine line : zone) {
-//                for (Component component : line.getComponents()) {
-//                    builder.expand(component.getChunk().getBounds());
-//                }
-//            }
-//            bounds.add(builder.getBounds());
-//        }
-//
-//        List<List<ComponentLine>> outputZones = new ArrayList<List<ComponentLine>>();
-//        mainFor:
-//        for (int i = 0; i < zones.size(); i++) {
-//            for (int j = 0; j < zones.size(); j++) {
-//                if (i == j || bounds.get(j) == null || bounds.get(i) == null) {
-//                    continue;
-//                }
-//                if (BxModelUtils.contains(bounds.get(j), bounds.get(i), tolerance)) {
-//                    zones.get(j).addAll(zones.get(i));
-//                    bounds.set(i, null);
-//                    continue mainFor;
-//                }
-//            }
-//            outputZones.add(zones.get(i));
-//        }
-//        return outputZones;
-//    }
-
-
-    private List<Zone> mergeLinesInZones(List<Zone> zones,
-                                         double wordSpacing,
-                                         double minHorizontalDistance,
-                                         double maxHorizontalDistance,
-                                         double minVerticalDistance,
-                                         double maxVerticalDistance) {
-
-        List<Zone> outputZones = new ArrayList<>(zones.size());
-        for (Zone zone : zones) {
-            outputZones.add(mergeLinesInZone(zone, wordSpacing, minHorizontalDistance, maxHorizontalDistance, minVerticalDistance, maxVerticalDistance));
-        }
-        return outputZones;
-    }
-
-
-    private Zone mergeLinesInZone(Zone zone,
-                                  double wordSpacing,
-                                  double minHorizontalDistance,
-                                  double maxHorizontalDistance,
-                                  double minVerticalDistance,
-                                  double maxVerticalDistance) {
-
-        DisjointSets<Line> sets = new DisjointSets<>(zone.getLines());
-        for (int i = 0; i < zone.getLines().size(); i++) {
-            Line li = zone.getLines().get(i);
-            for (int j = i + 1; j < zone.getLines().size(); j++) {
-                Line lj = zone.getLines().get(j);
-                double hDist = li.horizontalDistance(lj);
-                double vDist = li.verticalDistance(lj);
-                if (minHorizontalDistance <= hDist && hDist <= maxHorizontalDistance && minVerticalDistance <= vDist && vDist <= maxVerticalDistance) {
-                    sets.union(li, lj);
-                } else if (minVerticalDistance <= vDist && vDist <= maxVerticalDistance && Math.abs(hDist - Math.min(li.getLength(), lj.getLength())) < 0.1) {
-                    boolean componentOverlap = false;
-                    int overlappingCount = 0;
-                    for (Character ci : li.getCharacters()) {
-                        for (Character cj : lj.getCharacters()) {
-                            double dist = ci.overlappingDistance(cj);
-                            if (dist > 2) {
-                                componentOverlap = true;
-                            }
-                            if (dist > 0) {
-                                overlappingCount++;
-                            }
-                        }
-                    }
-                    if (!componentOverlap && overlappingCount <= 2) {
-                        sets.union(li, lj);
-                    }
-                }
-            }
-        }
-        List<Line> outputZone = new ArrayList<>();
-        for (Set<Line> group : sets) {
-            List<Character> components = new ArrayList<>();
-            for (Line line : group) {
-                components.addAll(line.getCharacters());
-            }
-            components.sort(Comparator.comparingDouble(Character::getX));
-
-            outputZone.add(new Line(components, wordSpacing));
-        }
-        return new Zone(outputZone);
-    }
-
-}
@@ -1,32 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-public class AngleFilter {
-
-    protected double lowerAngle;
-    protected double upperAngle;
-
-
-    public AngleFilter(double lowerAngle, double upperAngle) {
-
-        if (lowerAngle < -Math.PI / 2) {
-            lowerAngle += Math.PI;
-        }
-        if (upperAngle >= Math.PI / 2) {
-            upperAngle -= Math.PI;
-        }
-
-        this.lowerAngle = lowerAngle;
-        this.upperAngle = upperAngle;
-    }
-
-
-    public boolean matches(Neighbor neighbor) {
-
-        if (lowerAngle <= upperAngle) {
-            return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
-        } else {
-            return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
-        }
-    }
-
-}
@@ -1,48 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.awt.geom.Rectangle2D;
-
-import lombok.Data;
-
-@Data
-public abstract class BoundingBox {
-
-    private Rectangle2D bBox;
-
-
-    public double getX() {
-
-        return bBox.getX();
-    }
-
-
-    public double getY() {
-
-        return bBox.getY();
-    }
-
-
-    public double getWidth() {
-
-        return bBox.getWidth();
-    }
-
-
-    public double getHeight() {
-
-        return bBox.getHeight();
-    }
-
-
-    public double getArea() {
-
-        return (bBox.getHeight() * bBox.getWidth());
-    }
-
-
-    public boolean contains(Rectangle2D contained, double tolerance) {
-
-        return bBox.getX() <= contained.getX() + tolerance && bBox.getY() <= contained.getY() + tolerance && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
-    }
-
-}
@@ -1,84 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-
-import lombok.Data;
-
-@Data
-public class Character {
-
-    private final double x;
-    private final double y;
-    private final RedTextPosition textPosition;
-
-    private List<Neighbor> neighbors = new ArrayList<>();
-
-
-    public Character(RedTextPosition chunk) {
-
-        this.x = chunk.getXDirAdj() + chunk.getWidthDirAdj() / 2;
-        this.y = chunk.getYDirAdj() + chunk.getHeightDir() / 2;
-        this.textPosition = chunk;
-    }
-
-
-    public double getHeight() {
-
-        return textPosition.getHeightDir();
-    }
-
-
-    public double distance(Character character) {
-
-        double dx = getX() - character.getX();
-        double dy = getY() - character.getY();
-        return Math.sqrt(dx * dx + dy * dy);
-    }
-
-
-    public double horizontalDistance(Character character) {
-
-        return Math.abs(getX() - character.getX());
-    }
-
-
-    public double verticalDistance(Character character) {
-
-        return Math.abs(getY() - character.getY());
-    }
-
-
-    public double overlappingDistance(Character other) {
-
-        double[] xs = new double[4];
-        double s = Math.sin(-0), c = Math.cos(-0);
-        xs[0] = c * x - s * y;
-        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
-        xs[2] = c * other.x - s * other.y;
-        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
-        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
-        Arrays.sort(xs);
-        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
-    }
-
-
-    public void setNeighbors(List<Neighbor> neighbors) {
-
-        this.neighbors = neighbors;
-    }
-
-
-    public double angle(Character character) {
-
-        if (getX() > character.getX()) {
-            return Math.atan2(getY() - character.getY(), getX() - character.getX());
-        } else {
-            return Math.atan2(character.getY() - getY(), character.getX() - getX());
-        }
-    }
-
-}
@@ -1,194 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.util.AbstractSet;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-public class DisjointSets<E> implements Iterable<Set<E>> {
-
-    private final Map<E, Entry<E>> map = new HashMap<>();
-
-
-    public DisjointSets(Collection<? extends E> collection) {
-
-        for (E element : collection) {
-            map.put(element, new Entry<E>(element));
-        }
-    }
-
-
-    public boolean areTogether(E e1, E e2) {
-
-        return map.get(e1).findRepresentative() == map.get(e2).findRepresentative();
-    }
-
-
-    public void union(E e1, E e2) {
-
-        Entry<E> r1 = map.get(e1).findRepresentative();
-        Entry<E> r2 = map.get(e2).findRepresentative();
-        if (r1 != r2) {
-            if (r1.size <= r2.size) {
-                r2.mergeWith(r1);
-            } else {
-                r1.mergeWith(r2);
-            }
-        }
-    }
-
-
-    @Override
-    public Iterator<Set<E>> iterator() {
-
-        return new Iterator<>() {
-
-            private final Iterator<Entry<E>> iterator = map.values().iterator();
-            private Entry<E> nextRepresentative;
-
-            {
-                findNextRepresentative();
-            }
-
-            @Override
-            public boolean hasNext() {
-
-                return nextRepresentative != null;
-            }
-
-
-            @Override
-            public Set<E> next() {
-
-                if (nextRepresentative == null) {
-                    throw new NoSuchElementException();
-                }
-                Set<E> result = nextRepresentative.asSet();
-                findNextRepresentative();
-                return result;
-            }
-
-
-            private void findNextRepresentative() {
-
-                while (iterator.hasNext()) {
-                    Entry<E> candidate = iterator.next();
-                    if (candidate.isRepresentative()) {
-                        nextRepresentative = candidate;
-                        return;
-                    }
-                }
-                nextRepresentative = null;
-            }
-
-
-            @Override
-            public void remove() {
-
-                throw new UnsupportedOperationException();
-            }
-
-        };
-    }
-
-
-    private static class Entry<E> {
-
-        private int size = 1;
-        private final E value;
-        private Entry<E> parent = this;
-        private Entry<E> next = null;
-        private Entry<E> last = this;
-
-
-        Entry(E value) {
-
-            this.value = value;
-        }
-
-
-        void mergeWith(Entry<E> otherRepresentative) {
-
-            size += otherRepresentative.size;
-            last.next = otherRepresentative;
-            last = otherRepresentative.last;
-            otherRepresentative.parent = this;
-        }
-
-
-        Entry<E> findRepresentative() {
-
-            Entry<E> representative = parent;
-            while (representative.parent != representative) {
-                representative = representative.parent;
-            }
-            for (Entry<E> entry = this; entry != representative; ) {
-                Entry<E> nextEntry = entry.parent;
-                entry.parent = representative;
-                entry = nextEntry;
-            }
-            return representative;
-        }
-
-
-        boolean isRepresentative() {
-
-            return parent == this;
-        }
-
-
-        Set<E> asSet() {
-
-            return new AbstractSet<E>() {
-
-                @Override
-                public Iterator<E> iterator() {
-
-                    return new Iterator<E>() {
-
-                        private Entry<E> nextEntry = findRepresentative();
-
-
-                        @Override
-                        public boolean hasNext() {
-
-                            return nextEntry != null;
-                        }
-
-
-                        @Override
-                        public E next() {
-
-                            if (nextEntry == null) {
-                                throw new NoSuchElementException();
-                            }
-                            E result = nextEntry.value;
-                            nextEntry = nextEntry.next;
-                            return result;
-                        }
-
-
-                        @Override
-                        public void remove() {
-
-                            throw new UnsupportedOperationException();
-                        }
-
-                    };
-                }
-
-
-                @Override
-                public int size() {
-
-                    return findRepresentative().size;
-                }
-            };
-        }
-
-    }
-
-}
@@ -1,91 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-public class Histogram {
-
-    private static final double EPSILON = 1.0e-6;
-    private final double min;
-    private final double resolution;
-    private double[] frequencies;
-
-
-    public Histogram(double minValue, double maxValue, double resolution) {
-
-        this.min = minValue - EPSILON;
-        double delta = maxValue - minValue + 2 * EPSILON;
-        int size = Math.max(1, (int) Math.round((maxValue - minValue) / resolution));
-        this.resolution = delta / size;
-        this.frequencies = new double[size];
-    }
-
-
-    public void kernelSmooth(double[] kernel) {
-
-        double[] newFrequencies = new double[frequencies.length];
-        int shift = (kernel.length - 1) / 2;
-        for (int i = 0; i < kernel.length; i++) {
-            int jStart = Math.max(0, i - shift);
-            int jEnd = Math.min(frequencies.length, frequencies.length + i - shift);
-            for (int j = jStart; j < jEnd; j++) {
-                newFrequencies[j - i + shift] += kernel[i] * frequencies[j];
-            }
-        }
-        frequencies = newFrequencies;
-    }
-
-
-    public double[] createGaussianKernel(double length, double stdDeviation) {
-
-        int r = (int) Math.round(length / resolution) / 2;
-        stdDeviation /= resolution;
-
-        int size = 2 * r + 1;
-        double[] kernel = new double[size];
-        double sum = 0;
-        double b = 2 * stdDeviation * stdDeviation;
-        double a = 1 / Math.sqrt(Math.PI * b);
-        for (int i = 0; i < size; i++) {
-            kernel[i] = a * Math.exp(-(i - r) * (i - r) / b);
-            sum += kernel[i];
-        }
-        for (int i = 0; i < size; i++) {
-            kernel[i] /= sum;
-        }
-        return kernel;
-    }
-
-
-    public void gaussianSmooth(double windowLength, double stdDeviation) {
-
-        kernelSmooth(createGaussianKernel(windowLength, stdDeviation));
-    }
-
-
-    public void add(double value) {
-
-        frequencies[(int) ((value - min) / resolution)] += 1.0;
-    }
-
-
-    public int getSize() {
-
-        return frequencies.length;
-    }
-
-
-    public double getPeakValue() {
-
-        int peakIndex = 0;
-        for (int i = 1; i < frequencies.length; i++) {
-            if (frequencies[i] > frequencies[peakIndex]) {
-                peakIndex = i;
-            }
-        }
-        int peakEndIndex = peakIndex + 1;
-        final double EPS = 0.0001;
-        while (peakEndIndex < frequencies.length && Math.abs(frequencies[peakEndIndex] - frequencies[peakIndex]) < EPS) {
-            peakEndIndex++;
-        }
-        return ((double) peakIndex + peakEndIndex) / 2 * resolution + min;
-    }
-
-}
@@ -1,165 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-
-import lombok.Data;
-
-@Data
-public class Line extends BoundingBox {
-
-    private static final double WORD_DISTANCE_MULTIPLIER = 0.2;
-
-    private final double x0;
-    private final double y0;
-
-    private final double x1;
-    private final double y1;
-
-    private final double height;
-
-    private final List<Character> characters;
-    private final List<TextPositionSequence> words = new ArrayList<>();
-
-
-    public Line(List<Character> characters, double wordSpacing) {
-
-        this.characters = characters;
-
-        if (characters.size() >= 2) {
-            // linear regression
-            double sx = 0.0, sxx = 0.0, sxy = 0.0, sy = 0.0;
-            for (Character character : characters) {
-                sx += character.getX();
-                sxx += character.getX() * character.getX();
-                sxy += character.getX() * character.getY();
-                sy += character.getY();
-            }
-            double b = (characters.size() * sxy - sx * sy) / (characters.size() * sxx - sx * sx);
-            double a = (sy - b * sx) / characters.size();
-
-            this.x0 = characters.get(0).getX();
-            this.y0 = a + b * this.x0;
-            this.x1 = characters.get(characters.size() - 1).getX();
-            this.y1 = a + b * this.x1;
-        } else {
-            Character character = characters.get(0);
-            double dx = character.getTextPosition().getWidthDirAdj() / 3;
-            double dy = dx * Math.tan(0);
-            this.x0 = character.getX() - dx;
-            this.x1 = character.getX() + dx;
-            this.y0 = character.getY() - dy;
-            this.y1 = character.getY() + dy;
-        }
-        height = computeHeight();
-        computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
-        buildBBox();
-    }
-
-
-    public double getAngle() {
-
-        return Math.atan2(y1 - y0, x1 - x0);
-    }
-
-
-    public double getLength() {
-
-        return Math.sqrt((x0 - x1) * (x0 - x1) + (y0 - y1) * (y0 - y1));
-    }
-
-
-    private double computeHeight() {
-
-        double sum = 0.0;
-        for (Character component : characters) {
-            sum += component.getHeight();
-        }
-        return sum / characters.size();
-    }
-
-
-    public double angularDifference(Line j) {
-
-        double diff = Math.abs(getAngle() - j.getAngle());
-        if (diff <= Math.PI / 2) {
-            return diff;
-        } else {
-            return Math.PI - diff;
-        }
-    }
-
-
-    public double horizontalDistance(Line other) {
-
-        double[] xs = new double[4];
-        xs[0] = x0;
-        xs[1] = x1;
-        xs[2] = other.x0;
-        xs[3] = other.x1;
-        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
-        Arrays.sort(xs);
-        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
-    }
-
-
-    public double verticalDistance(Line other) {
-
-        double ym = (y0 + y1) / 2;
-        double yn = (other.y0 + other.y1) / 2;
-        return Math.abs(ym - yn) / Math.sqrt(1);
-    }
-
-
-    private void computeWords(double wordSpacing) {
-
-        TextPositionSequence word = new TextPositionSequence();
-        Character previous = null;
-        for (Character current : characters) {
-            if (previous != null) {
-                double dist = current.getTextPosition().getXDirAdj() - previous.getTextPosition().getXDirAdj() - previous.getTextPosition().getWidthDirAdj();
-                if (dist > wordSpacing) {
-                    words.add(word);
-                    word = new TextPositionSequence();
-                }
-            }
-            word.getTextPositions().add(current.getTextPosition());
-            previous = current;
-        }
-        words.add(word);
-    }
-
-
-    private void buildBBox() {
-
-        double minX = Double.POSITIVE_INFINITY;
-        double minY = Double.POSITIVE_INFINITY;
-        double maxX = Double.NEGATIVE_INFINITY;
-        double maxY = Double.NEGATIVE_INFINITY;
-
-        for (Character character : characters) {
-
-            minX = Math.min(minX, character.getTextPosition().getXDirAdj());
-            minY = Math.min(minY, character.getTextPosition().getYDirAdj());
-            maxX = Math.max(maxX, character.getTextPosition().getXDirAdj() + character.getTextPosition().getWidthDirAdj());
-            maxY = Math.max(maxY, character.getTextPosition().getYDirAdj() + character.getTextPosition().getHeightDir());
-
-        }
-
-        this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
-    }
-
-
-    public String toString() {
-
-        StringBuilder sb = new StringBuilder();
-        words.forEach(word -> sb.append(word.toString()).append(" "));
-        return sb.toString().trim();
-    }
-
-}
-
@@ -1,36 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import lombok.Getter;
-
-public class Neighbor {
-
-    @Getter
-    private final double distance;
-    @Getter
-    private final double angle;
-    private final Character originCharacter;
-    @Getter
-    private final Character character;
-
-
-    public Neighbor(Character neighbor, Character origin) {
-
-        this.distance = neighbor.distance(origin);
-        this.angle = neighbor.angle(origin);
-        this.character = neighbor;
-        this.originCharacter = origin;
-    }
-
-
-    public double getHorizontalDistance() {
-
-        return character.horizontalDistance(originCharacter);
-    }
-
-
-    public double getVerticalDistance() {
-
-        return character.verticalDistance(originCharacter);
-    }
-
-}
@@ -1,50 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Comparator;
-import java.util.List;
-
-import lombok.Data;
-
-@Data
-public class Zone extends BoundingBox {
-
-    private List<Line> lines;
-
-
-    public Zone(List<Line> lines) {
-
-        lines.sort(Comparator.comparingDouble(Line::getY));
-        this.lines = lines;
-        buildBBox();
-    }
-
-
-    public void buildBBox() {
-
-        double minX = Double.POSITIVE_INFINITY;
-        double minY = Double.POSITIVE_INFINITY;
-        double maxX = Double.NEGATIVE_INFINITY;
-        double maxY = Double.NEGATIVE_INFINITY;
-
-        for (Line line : lines) {
-
-            minX = Math.min(minX, line.getX());
-            minY = Math.min(minY, line.getY());
-            maxX = Math.max(maxX, line.getX() + line.getWidth());
-            maxY = Math.max(maxY, line.getY() + line.getHeight());
-
-        }
-
-        this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
-    }
-
-
-    public String toString() {
-
-        StringBuilder sb = new StringBuilder();
-        lines.forEach(line -> sb.append(line.toString()).append("\n"));
-        return sb.toString().trim();
-    }
-
-}
@@ -1,50 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-
-@Service
-public class LineBuilderService {
-
-    private static final double CHARACTER_SPACING_DISTANCE_MULTIPLIER = 3.5;
-    private static final double MAX_VERTICAL_CHARACTER_DISTANCE = 0.67;
-    private static final double ANGLE_TOLERANCE = Math.PI / 6;
-
-
-    public List<Line> buildLines(List<Character> characters, double characterSpacing, double lineSpacing) {
-
-        double maxHorizontalDistance = characterSpacing * CHARACTER_SPACING_DISTANCE_MULTIPLIER;
-        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_CHARACTER_DISTANCE;
-
-        DisjointSets<Character> sets = new DisjointSets<>(characters);
-        AngleFilter filter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
-
-        characters.forEach(character -> {
-            character.getNeighbors().forEach(neighbor -> {
-                double x = neighbor.getHorizontalDistance() / maxHorizontalDistance;
-                double y = neighbor.getVerticalDistance() / maxVerticalDistance;
-                if (filter.matches(neighbor) && Math.pow(x, 2) + Math.pow(y, 2) <= 1) {
-                    sets.union(character, neighbor.getCharacter());
-                }
-            });
-        });
-
-        List<Line> lines = new ArrayList<>();
-        sets.forEach(group -> {
-            List<Character> lineComponents = new ArrayList<>(group);
-            lineComponents.sort(Comparator.comparingDouble(Character::getX));
-            lines.add(new Line(lineComponents, characterSpacing));
-        });
-
-        return lines;
-    }
-
-}
@@ -1,78 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
-
-@Service
-public class NearestNeighbourService {
-
-    private static final int NUMBER_OF_NEIGHBOURS = 8;
-    private static final double STEP = 16.0;
-
-
-    public void findNearestNeighbors(List<Character> characters) {
-
-        if (characters.isEmpty()) {
-            return;
-        }
-
-        characters.sort(Comparator.comparingDouble(Character::getX));
-
-        int maxNeighborCount = NUMBER_OF_NEIGHBOURS;
-        if (characters.size() <= NUMBER_OF_NEIGHBOURS) {
-            maxNeighborCount = characters.size() - 1;
-        }
-
-        for (int i = 0; i < characters.size(); i++) {
-
-            List<Neighbor> candidates = new ArrayList<>();
-
-            int start = i;
-            int end = i + 1;
-
-            double distance = Double.POSITIVE_INFINITY;
-
-            for (double searchDistance = 0; searchDistance < distance; ) {
-
-                searchDistance += STEP;
-                boolean newCandidatesFound = false;
-
-                while (start > 0 && characters.get(i).getX() - characters.get(start - 1).getX() < searchDistance) {
-                    start--;
-                    candidates.add(new Neighbor(characters.get(start), characters.get(i)));
-                    clearLeastDistant(candidates, maxNeighborCount);
-                    newCandidatesFound = true;
-                }
-
-                while (end < characters.size() && characters.get(end).getX() - characters.get(i).getX() < searchDistance) {
-                    candidates.add(new Neighbor(characters.get(end), characters.get(i)));
-                    clearLeastDistant(candidates, maxNeighborCount);
-                    end++;
-                    newCandidatesFound = true;
-                }
-
-                if (newCandidatesFound && candidates.size() >= maxNeighborCount) {
-                    distance = candidates.get(maxNeighborCount - 1).getDistance();
-                }
-            }
-            clearLeastDistant(candidates, maxNeighborCount);
-            characters.get(i).setNeighbors(new ArrayList<>(candidates));
-        }
-    }
-
-
-    private void clearLeastDistant(List<Neighbor> candidates, int maxNeighborCount) {
-
-        if (candidates.size() > maxNeighborCount) {
-            candidates.sort(Comparator.comparingDouble(Neighbor::getDistance));
-            candidates.remove(candidates.remove(candidates.size() - 1));
-        }
-    }
-
-}
@@ -1,99 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.ListIterator;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
-
-@Service
-public class ReadingOrderService {
-
-    private static final double THRESHOLD = 1;
-
-
-    public List<Zone> resolve(List<Zone> zones, boolean yxOrder) {
-
-        if (zones.isEmpty() || zones.size() == 1) {
-            return zones;
-        }
-
-        if (yxOrder) {
-            zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                    .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-            return zones;
-        }
-
-        return resolveMultiColumnReadingOder(zones);
-    }
-
-
-    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
-
-        // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
-        // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
-
-        double minX = Double.POSITIVE_INFINITY;
-        double maxX = Double.NEGATIVE_INFINITY;
-
-        for (Zone zone : zones) {
-            if (zone.getX() < minX) {
-                minX = zone.getX();
-            }
-            if (zone.getX() + zone.getWidth() > maxX) {
-                maxX = zone.getX() + zone.getWidth();
-            }
-        }
-
-        double midLineXCoordinate = (minX + maxX) / 2;
-
-        List<Zone> leftOf = new ArrayList<>();
-        List<Zone> rightOf = new ArrayList<>();
-        List<Zone> middle = new ArrayList<>();
-        for (Zone zone : zones) {
-            if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
-                leftOf.add(zone);
-            } else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
-                rightOf.add(zone);
-            } else {
-                middle.add(zone);
-            }
-        }
-
-        leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
-                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
-
-        List<Zone> sortedZones = new ArrayList<>();
-        sortedZones.addAll(leftOf);
-        sortedZones.addAll(rightOf);
-
-        ListIterator<Zone> itty = middle.listIterator();
-
-        while (itty.hasNext()) {
-            Zone current = itty.next();
-            for (int i = 0; i < sortedZones.size(); i++) {
-                if (current.getY() < sortedZones.get(i).getY()) {
-                    sortedZones.add(i, current);
-                    itty.remove();
-                    break;
-                }
-            }
-        }
-
-        sortedZones.addAll(middle);
-
-        return sortedZones;
-    }
-
-}
@@ -1,56 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Histogram;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Neighbor;
-
-@Service
-public class SpacingService {
-
-    private static final double SPACING_HISTOGRAM_RESOLUTION = 0.5;
-    private static final double SPACING_HISTOGRAM_SMOOTHING_LENGTH = 2.5;
-    private static final double SPACING_HIST_SMOOTHING_STANDARD_DEVIATION = 0.5;
-    private static final double ANGLE_TOLERANCE = Math.PI / 6;
-
-
-    public double computeCharacterSpacing(List<Character> components) {
-
-        return computeSpacing(components, 0);
-    }
-
-
-    public double computeLineSpacing(List<Character> components) {
-
-        return computeSpacing(components, Math.PI / 2);
-    }
-
-
-    private double computeSpacing(List<Character> components, double angle) {
-
-        double maxDistance = Double.NEGATIVE_INFINITY;
-
-        for (Character component : components) {
-            for (Neighbor neighbor : component.getNeighbors()) {
-                maxDistance = Math.max(maxDistance, neighbor.getDistance());
-            }
-        }
-        Histogram histogram = new Histogram(0, maxDistance, SPACING_HISTOGRAM_RESOLUTION);
-        AngleFilter filter = new AngleFilter(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
-        for (Character component : components) {
-            for (Neighbor neighbor : component.getNeighbors()) {
-                if (filter.matches(neighbor)) {
-                    histogram.add(neighbor.getDistance());
-                }
-            }
-        }
-
-        histogram.gaussianSmooth(SPACING_HISTOGRAM_SMOOTHING_LENGTH, SPACING_HIST_SMOOTHING_STANDARD_DEVIATION);
-        return histogram.getPeakValue();
-    }
-
-}
@@ -1,90 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-
-@Service
-public class ZoneBuilderService {
-
-    private static final double MIN_HORIZONTAL_DISTANCE_MULTIPLIER = -0.5;
-    private static final double MAX_VERTICAL_DISTANCE_MULTIPLIER = 1.2;
-
-    private static final double MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER = -3.0;
-
-    private static final double MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER = 0.5;
-
-    private static final double MIN_LINE_SIZE_SCALE = 0.9;
-
-    private static final double MAX_LINE_SIZE_SCALE = 2.5;
-
-    private static final double ANGLE_TOLERANCE = Math.PI / 6;
-
-    public static final int MAX_ZONES = 300;
-
-
-    public List<Zone> buildZones(List<Line> lines, double characterSpacing, double lineSpacing) {
-
-        double minHorizontalDistance = characterSpacing * MIN_HORIZONTAL_DISTANCE_MULTIPLIER;
-        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_DISTANCE_MULTIPLIER;
-        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
-        double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;
-
-        DisjointSets<Line> sets = new DisjointSets<>(lines);
-
-        double meanHeight = calculateMeanHeight(lines);
-
-        lines.forEach(outerLine -> //
-                lines.forEach(innerLine -> {
-
-                    double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
-                    scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
-
-                    if (!sets.areTogether(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {
-
-                        double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
-                        double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
-
-                        if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance //
-                                || minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
-                            sets.union(outerLine, innerLine);
-                        }
-                    }
-                }));
-
-        List<Zone> zones = new ArrayList<>();
-        sets.forEach(group -> {
-            zones.add(new Zone(new ArrayList<>(group)));
-        });
-
-        if (zones.size() > MAX_ZONES) {
-            List<Line> oneZoneLines = new ArrayList<>();
-            for (Zone zone : zones) {
-                oneZoneLines.addAll(zone.getLines());
-            }
-            return List.of(new Zone(oneZoneLines));
-        }
-
-        return zones;
-    }
-
-
-    private double calculateMeanHeight(List<Line> lines) {
-
-        double meanHeight = 0.0;
-        double weights = 0.0;
-        for (Line line : lines) {
-            double weight = line.getLength();
-            meanHeight += line.getHeight() * weight;
-            weights += weight;
-        }
-        meanHeight /= weights;
-        return meanHeight;
-    }
-
-}
@@ -1,18 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils;
-
-public class DoubleUtils {
-
-    public static int compareDouble(double d1, double d2, double precision) {
-
-        if (Double.isNaN(d1) || Double.isNaN(d2)) {
-            return Double.compare(d1, d2);
-        }
-        if (precision == 0) {
-            precision = 1;
-        }
-        long i1 = Math.round(d1 / precision);
-        long i2 = Math.round(d2 / precision);
-        return Long.valueOf(i1).compareTo(i2);
-    }
-
-}
@@ -7,19 +7,19 @@ import static java.util.stream.Collectors.toList;
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
@@ -31,6 +31,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

@@ -95,14 +97,17 @@ public class DocumentGraphFactory {

        Rectangle2D position = image.getPosition();
        Page page = context.getPage(image.getPage());
-        Image imageNode = Image.builder()
+        var imageBuilder = Image.builder()
                .id(IdBuilder.buildId(Set.of(page), List.of(position)))
                .imageType(image.getImageType())
                .position(position)
                .transparent(image.isHasTransparency())
                .page(page)
-                .documentTree(context.getDocumentTree())
-                .build();
+                .documentTree(context.getDocumentTree());
+        if (image.isSourceByAi()) {
+            imageBuilder.engines(new HashSet<>(Set.of(LayoutEngine.AI)));
+        }
+        Image imageNode = imageBuilder.build();
        page.getMainBody().add(imageNode);

        List<Integer> tocId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode);
@@ -80,6 +80,7 @@ public class DocumentDataMapper {
                .treeId(toPrimitiveIntArray(entry.getTreeId()))
                .children(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
                .type(entry.getType())
+                .engines(entry.getNode().getEngines())
                .atomicBlockIds(atomicTextBlocks)
                .pageNumbers(entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new))
                .properties(properties)
@@ -17,6 +17,7 @@ import java.util.stream.Stream;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
@@ -63,43 +64,53 @@ public class LayoutGridService {
    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {

-        LayoutGrid layoutGrid = createLayoutGrid(document);
+        Visualizations layoutGrid = this.addLayoutGrid(document, layerVisibilityDefaultValue, false);
+        Visualizations visualLayoutGrid = this.addLayoutGrid(document, layerVisibilityDefaultValue, true);

-        viewerDocumentService.addVisualizationsOnPage(originFile,
-                destinationFile,
-                Visualizations.builder()
-                        .layer(ContentStreams.KNECON_LAYOUT)
-                        .visualizationsOnPages(layoutGrid.getVisualizationsPerPages())
-                        .layerVisibilityDefaultValue(layerVisibilityDefaultValue)
-                        .build());
+        viewerDocumentService.addVisualizationsOnPage(originFile, destinationFile, List.of(layoutGrid, visualLayoutGrid));
    }


-    private LayoutGrid createLayoutGrid(Document document) {
+    @SneakyThrows
+    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
+    public Visualizations addLayoutGrid(Document document, boolean layerVisibilityDefaultValue, boolean visualParsingGrid) {
+
+        LayoutGrid layoutGrid = createLayoutGrid(document, visualParsingGrid);
+
+        return Visualizations.builder()
+                .layer(visualParsingGrid ? ContentStreams.KNECON_VISUAL_PARSING : ContentStreams.KNECON_LAYOUT)
+                .visualizationsOnPages(layoutGrid.getVisualizationsPerPages())
+                .layerVisibilityDefaultValue(layerVisibilityDefaultValue)
+                .build();
+    }
+
+
+    private LayoutGrid createLayoutGrid(Document document, boolean visualParsingGrid) {

        LayoutGrid layoutGrid = new LayoutGrid(document.getNumberOfPages());
-        document.streamAllSubNodes().forEach(semanticNode -> {
-            Color color = switch (semanticNode.getType()) {
-                case PARAGRAPH -> PARAGRAPH_COLOR;
-                case TABLE -> TABLE_COLOR;
-                case SECTION -> SECTION_COLOR;
-                case HEADLINE -> HEADLINE_COLOR;
-                case HEADER, FOOTER -> HEADER_COLOR;
-                case IMAGE -> IMAGE_COLOR;
-                default -> null;
-            };
-            if (isNotSectionOrTableCellOrDocument(semanticNode)) {
-                addAsRectangle(semanticNode, layoutGrid, color);
-            }
-            if (semanticNode.getType().equals(NodeType.SECTION)) {
-                addSection(semanticNode, layoutGrid, color);
-            }
-            if (semanticNode.getType().equals(NodeType.TABLE)) {
-                Table table = (Table) semanticNode;
-                addInnerTableLines(table, layoutGrid);
-            }
-        });
-
+        document.streamAllSubNodes()
+                .filter(node -> (node.getEngines().contains(LayoutEngine.AI) && visualParsingGrid) || (node.getEngines().contains(LayoutEngine.ALGORITHM) && !visualParsingGrid))
+                .forEach(semanticNode -> {
+                    Color color = switch (semanticNode.getType()) {
+                        case PARAGRAPH -> PARAGRAPH_COLOR;
+                        case TABLE -> TABLE_COLOR;
+                        case SECTION -> SECTION_COLOR;
+                        case HEADLINE -> HEADLINE_COLOR;
+                        case HEADER, FOOTER -> HEADER_COLOR;
+                        case IMAGE -> IMAGE_COLOR;
+                        default -> null;
+                    };
+                    if (isNotSectionOrTableCellOrDocument(semanticNode)) {
+                        addAsRectangle(semanticNode, layoutGrid, color);
+                    }
+                    if (semanticNode.getType().equals(NodeType.SECTION)) {
+                        addSection(semanticNode, layoutGrid, color);
+                    }
+                    if (semanticNode.getType().equals(NodeType.TABLE)) {
+                        Table table = (Table) semanticNode;
+                        addInnerTableLines(table, layoutGrid);
+                    }
+                });
        return layoutGrid;
    }

@@ -1,270 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
-
-import java.awt.Color;
-import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.PDPageContentStream;
-import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
-import org.apache.pdfbox.util.Matrix;
-import org.springframework.core.io.ClassPathResource;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class PdfDraw {
-
-    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) throws IOException {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
-                        pageNumber,
-                        rectanglesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-
-    }
-
-
-    public static void drawRectanglesPerPageNumberedByLine(String filename, List<List<List<Rectangle2D>>> rectanglesPerPage, String tmpFileName) throws IOException {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                var rectanglesOnPage = rectanglesPerPage.get(pageNumber - 1);
-                for (int lineNumber = 0; lineNumber < rectanglesOnPage.size(); lineNumber++) {
-                    var rectanglesInLine = rectanglesOnPage.get(lineNumber);
-                    PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, rectanglesInLine, PdfVisualisationUtility.Options.builder().stroke(true).build());
-                    double y = Math.min(rectanglesInLine.get(0).getMinY(), rectanglesInLine.get(0).getMaxY());
-                    PdfVisualisationUtility.drawText(String.format("%d", lineNumber),
-                            pdDocument,
-                            new Point2D.Double(rectanglesInLine.get(0).getX() - (5 + (5 * countNumberOfDigits(lineNumber))), y + 2),
-                            pageNumber,
-                            PdfVisualisationUtility.Options.builder().stroke(true).build());
-                }
-            }
-            pdDocument.save(out);
-        }
-
-    }
-
-
-    private static int countNumberOfDigits(int num) {
-
-        int final_num = num;
-        if (final_num == 0) {
-            return 1;
-        }
-        int count = 0;
-        for (; final_num != 0; final_num /= 10) {
-            count++;
-        }
-        return count;
-    }
-
-
-    public static void drawDocumentGraph(PDDocument document, Document documentGraph) {
-
-        documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry));
-    }
-
-
-    public static void drawNode(PDDocument document, DocumentTree.Entry entry) {
-
-        Options options = buildStandardOptionsForNodes(entry);
-
-        drawBBoxAndLabelAndNumberOnPage(document, entry, options);
-
-    }
-
-
-    public static void drawTextBlock(PDDocument document, TextBlock textBlock, Options options) {
-
-        textBlock.getAtomicTextBlocks().forEach(atb -> drawAtomicTextBlock(document, atb, options));
-    }
-
-
-    public static void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, Options options) {
-
-        drawRectangle2DList(document, atomicTextBlock.getPage().getNumber(), atomicTextBlock.getPositions().stream().toList(), options);
-
-    }
-
-
-    @SneakyThrows
-    private static void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, Options options, boolean rotate) {
-
-        var pdPage = document.getPage(pageNumber - 1);
-        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
-
-        contentStream.setNonStrokingColor(options.getStrokeColor());
-        contentStream.setLineWidth(options.getStrokeWidth());
-
-        contentStream.beginText();
-        if (rotate) {
-            contentStream.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(15), (float) location.getX(), (float) location.getY()));
-        } else {
-            contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
-        }
-        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
-        contentStream.showText(string);
-        contentStream.endText();
-        contentStream.close();
-    }
-
-
-    @SneakyThrows
-    public static void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, Options options) {
-
-        var pdPage = document.getPage(pageNumber - 1);
-        drawRectangle2DList(document, rectCollection, options, pdPage);
-    }
-
-
-    private static void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, Options options, PDPage pdPage) throws IOException {
-
-        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
-
-        contentStream.setStrokingColor(options.getStrokeColor());
-        contentStream.setNonStrokingColor(options.getFillColor());
-        contentStream.setLineWidth(options.getStrokeWidth());
-
-        for (var r : rectCollection) {
-            contentStream.addRect((float) r.getMinX(), (float) r.getMinY(), (float) r.getWidth(), (float) r.getHeight());
-
-            if (options.isStroke() && options.isFill()) {
-                contentStream.fillAndStroke();
-            } else if (options.isStroke()) {
-                contentStream.stroke();
-            } else if (options.isFill()) {
-                contentStream.fill();
-            }
-        }
-        contentStream.close();
-    }
-
-
-    @SneakyThrows
-    public static void drawRectanglesAndLinesPerPage(String filename, List<List<Rectangle2D>> list, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-//                PdfVisualisationUtility.drawLine2DList(pdDocument,
-//                        pageNumber,
-//                        list.get(pageNumber - 1),
-//                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
-                        pageNumber,
-                        rectanglesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-    }
-
-
-    @SneakyThrows
-    public static void drawLinesPerPage(String filename, List<List<Ruling>> linesPerPage, String tmpFileName) {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                PdfVisualisationUtility.drawLine2DList(pdDocument,
-                        pageNumber,
-                        linesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-    }
-
-
-    @Builder
-    @AllArgsConstructor
-    @Getter
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    public static class Options {
-
-        boolean stroke;
-        @Builder.Default
-        Color strokeColor = Color.BLACK;
-        @Builder.Default
-        float strokeWidth = 1f;
-
-        boolean fill;
-        @Builder.Default
-        Color fillColor = Color.BLACK;
-
-    }
-
-
-    private static Options buildStandardOptionsForNodes(DocumentTree.Entry entry) {
-
-        return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
-            case DOCUMENT -> Color.LIGHT_GRAY;
-            case HEADER, FOOTER -> Color.GREEN;
-            case PARAGRAPH -> Color.BLUE;
-            case HEADLINE -> Color.RED;
-            case SECTION -> Color.BLACK;
-            case TABLE -> Color.ORANGE;
-            case TABLE_CELL -> Color.GRAY;
-            case IMAGE -> Color.MAGENTA;
-        }).build();
-    }
-
-
-    private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, Options options) {
-
-        Map<Page, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
-        for (Page page : rectanglesPerPage.keySet()) {
-            Rectangle2D rectangle2D = rectanglesPerPage.get(page);
-            if (entry.getType() == NodeType.SECTION) {
-                rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
-            }
-            drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options);
-            drawText(buildString(entry),
-                    document,
-                    new Point2D.Double(rectangle2D.getMinX(), rectangle2D.getMaxY() + 2),
-                    page.getNumber(),
-                    options,
-                    entry.getType() == NodeType.TABLE_CELL);
-        }
-    }
-
-
-    private static String buildString(DocumentTree.Entry entry) {
-
-        return entry.getNode().getNumberOnPage() + ": " + entry.getTreeId() + ": " + entry.getType();
-    }
-
-}
@@ -30,8 +30,6 @@ public class TableMergingUtility {
            if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() && !hasTableHeader(consecutiveTable) && outerBoundaryAlignsX(originalTablePageBlock,
                    consecutiveTable)) {
                consecutiveTablesWithSameColCountAndHeaders.add(consecutiveTable);
-            } else {
-                break;
            }
        }
        return Stream.concat(Stream.of(originalTablePageBlock), consecutiveTablesWithSameColCountAndHeaders.stream()).toList();
@@ -27,6 +27,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
@@ -50,6 +51,7 @@ public class BdrJsonBuildTest extends AbstractTest {
                file,
                new ImageServiceResponse(),
                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
                file.toString()));
    }

@@ -33,6 +33,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;

@@ -98,6 +99,7 @@ public class HeadlinesGoldStandardIntegrationTest {
                pdfFileResource.getFile(),
                new ImageServiceResponse(),
                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
                filePath));

        var foundHeadlines = documentGraph.streamAllSubNodes()
@@ -25,9 +25,7 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
    @SneakyThrows
    public void testLayoutParserEndToEnd() {

-        String s = "（";
-        String s1 = "）";
-        prepareStorage("files/Minimal Examples/WrongOrderPage1.pdf");
+        prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
@@ -16,6 +16,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
@@ -58,6 +59,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
                filename.toFile(),
                new ImageServiceResponse(),
                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
                filename.toFile().toString()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
@@ -12,6 +12,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
@@ -25,7 +26,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
    @SneakyThrows
    public void testViewerDocument() {

-        String fileName = "files/Plenarprotokoll 1 (keine Druchsache!) (1).pdf";
+        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";

        var documentFile = new ClassPathResource(fileName).getFile();
@@ -53,10 +54,10 @@ public class ViewerDocumentTest extends BuildDocumentTest {
        var documentFile = new ClassPathResource(fileName).getFile();

        var classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
-                documentFile,
-                new ImageServiceResponse(),
-                tableResponse,
-                Path.of(fileName).getFileName().toFile().toString());
+                                                                       documentFile,
+                                                                       new ImageServiceResponse(),
+                                                                       tableResponse,
+                                                                       new VisualLayoutParsingResponse(),Path.of(fileName).getFileName().toFile().toString());
        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
        LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
@@ -65,3 +66,4 @@ public class ViewerDocumentTest extends BuildDocumentTest {
    }

 }
+
@@ -29,10 +29,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Im
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
@@ -50,12 +49,6 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @Autowired
    private RedactManagerClassificationService redactManagerClassificationService;

-    @Autowired
-    private CvTableParsingAdapter cvTableParsingAdapter;
-
-    @Autowired
-    private ImageServiceResponseAdapter imageServiceResponseAdapter;
-
    @Autowired
    private SectionsBuilderService sectionsBuilderService;

@@ -64,10 +57,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {

        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                originDocument,
-                new ImageServiceResponse(),
-                tableServiceResponse,
-                "document");
+                                                                                          originDocument,
+                                                                                          new ImageServiceResponse(),
+                                                                                          tableServiceResponse,
+                                                                                          new VisualLayoutParsingResponse(),
+                                                                                          "document");

        redactManagerClassificationService.classifyDocument(classificationDocument);

@@ -87,11 +81,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @Test
    public void tablesToHtmlDebugger() throws IOException {

-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T5 VV-640252-Page16.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

-        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");
+        toHtml(document, "/tmp/T5.html");

    }

@@ -109,6 +103,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @Disabled
    @Test
    public void testScanRotationBorderIsIgnored() throws IOException {

@@ -117,8 +112,16 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile(), tableServiceResponse);
-        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+        assertThat(document.getSections()
+                           .stream()
+                           .flatMap(paragraph -> paragraph.getTables()
+                                   .stream())
+                           .collect(Collectors.toList())).isNotEmpty();
+        var tables = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList();

        // Quality of the table parsing is not good, because the file is rotated at scanning.
        // We only asset that the table border is not the page border.
@@ -140,12 +143,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        imageServiceResponse.getData()
                .forEach(imageMetadata -> images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
                        .add(new ClassifiedImage(new Rectangle2D.Double(imageMetadata.getPosition().getX1(),
-                                imageMetadata.getPosition().getY1(),
-                                imageMetadata.getGeometry().getWidth(),
-                                imageMetadata.getGeometry().getHeight()),
-                                ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)),
-                                imageMetadata.isAlpha(),
-                                imageMetadata.getPosition().getPageNumber())));
+                                                                        imageMetadata.getPosition().getY1(),
+                                                                        imageMetadata.getGeometry().getWidth(),
+                                                                        imageMetadata.getGeometry().getHeight()),
+                                                 ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)),
+                                                 imageMetadata.isAlpha(),
+                                                 imageMetadata.getPosition().getPageNumber())));

        System.out.println("object");
    }
@@ -157,11 +160,22 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        assertThat(document.getSections()
+                           .stream()
+                           .flatMap(paragraph -> paragraph.getTables()
+                                   .stream())
+                           .collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock table = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(0);
        assertThat(table.getColCount()).isEqualTo(6);
        assertThat(table.getRowCount()).isEqualTo(13);
-        assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
+        assertThat(table.getRows()
+                           .stream()
+                           .mapToInt(List::size).sum()).isEqualTo(6 * 13);
    }


@@ -171,15 +185,37 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        assertThat(document.getSections()
+                           .stream()
+                           .flatMap(paragraph -> paragraph.getTables()
+                                   .stream())
+                           .collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
        assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(1);
        assertThat(secondTable.getColCount()).isEqualTo(8);
        assertThat(secondTable.getRowCount()).isEqualTo(2);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
-        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
+                .get(0)
+                .stream()
+                .map(Collections::singletonList)
+                .collect(Collectors.toList());
+        assertThat(secondTable.getRows()
+                           .stream()
+                           .allMatch(row -> row.stream()
+                                   .map(Cell::getHeaderCells)
+                                   .toList().equals(firstTableHeaderCells))).isTrue();
    }


@@ -189,15 +225,37 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        assertThat(document.getSections()
+                           .stream()
+                           .flatMap(paragraph -> paragraph.getTables()
+                                   .stream())
+                           .collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(0);
        assertThat(firstTable.getColCount()).isEqualTo(9);
        assertThat(firstTable.getRowCount()).isEqualTo(5);
-        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(1);
        assertThat(secondTable.getColCount()).isEqualTo(9);
        assertThat(secondTable.getRowCount()).isEqualTo(6);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
-        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
+                .get(firstTable.getRowCount() - 1)
+                .stream()
+                .map(Cell::getHeaderCells)
+                .collect(Collectors.toList());
+        assertThat(secondTable.getRows()
+                           .stream()
+                           .allMatch(row -> row.stream()
+                                   .map(Cell::getHeaderCells)
+                                   .toList().equals(firstTableHeaderCells))).isTrue();
    }


@@ -207,19 +265,41 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
-        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        assertThat(document.getSections()
+                           .stream()
+                           .flatMap(paragraph -> paragraph.getTables()
+                                   .stream())
+                           .collect(Collectors.toList())).isNotEmpty();
+        TablePageBlock firstTable = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
        assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(1);
        assertThat(secondTable.getColCount()).isEqualTo(8);
        assertThat(secondTable.getRowCount()).isEqualTo(6);
-        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
-        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
+        List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
+                .get(0)
+                .stream()
+                .map(Collections::singletonList)
+                .collect(Collectors.toList());
+        assertThat(secondTable.getRows()
+                           .stream()
+                           .allMatch(row -> row.stream()
+                                   .map(Cell::getHeaderCells)
+                                   .toList().equals(firstTableHeaderCells))).isTrue();
    }


-    @Test // Non-sense test
+    @Test
    public void testDoc56Page170() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
@@ -230,8 +310,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        validateTable(document, 0, 1, 1, 0, 0);
        validateTable(document, 1, 2, 2, 0, 0);
-        validateTable(document, 2, 6, 20, 0, 0);
-        validateTable(document, 3, 7, 31, 0, 0);
+        validateTable(document, 2, 4, 19, 12, 0);
+        validateTable(document, 3, 2, 12, 0, 0);

    }

@@ -265,29 +345,30 @@ public class PdfSegmentationServiceTest extends AbstractTest {
        validateTable(document, 0, 8, 8, 0, 0);

        List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
-                        "Author, date",
-                        "Study title",
-                        "Analytical method Author, date, No.",
-                        "Technique, LOQ of the method, validated working range",
-                        "Method meets analytical validation criteria",
-                        "Remarks (in case validation criteria are not met)",
-                        "Acceptability of the method"),
-                Arrays.asList("Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
-                Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
-                        "Evans P.G. 2001 TMJ4569B, VV-323245",
-                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
-                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
-                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
-                        "Y",
-                        "N/A",
-                        "Y"));
+                                                                "Author, date",
+                                                                "Study title",
+                                                                "Analytical method Author, date, No.",
+                                                                "Technique, LOQ of the method, validated working range",
+                                                                "Method meets analytical validation criteria",
+                                                                "Remarks (in case validation criteria are not met)",
+                                                                "Acceptability of the method"),
+                                                  Arrays.asList(
+                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                                                          "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
+                                                  Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
+                                                                "Evans P.G. 2001 TMJ4569B, VV-323245",
+                                                                "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
+                                                                "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
+                                                                "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
+                                                                "Y",
+                                                                "N/A",
+                                                                "Y"));

        validateTable(document, 0, values);

@@ -579,10 +660,109 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @Test
+    public void testT0() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T0 TableWithMergedCells.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 6, 8, 0, 0);
+    }
+
+
+    @Test
+    public void testT1() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T1 MultipleNestedTable.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 3, 3, 0, 0);
+        validateTable(document, 1, 3, 5, 2, 0);
+        validateTable(document, 2, 3, 3, 1, 0);
+        validateTable(document, 3, 3, 3, 0, 0);
+
+    }
+
+
+    @Test
+    public void testT2() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T2 MultipleTables.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 6);
+
+        validateTable(document, 0, 5, 5, 0, 0);
+        validateTable(document, 1, 5, 6, 0, 0);
+        validateTable(document, 2, 5, 5, 0, 0);
+        validateTable(document, 3, 5, 5, 0, 0);
+        validateTable(document, 4, 5, 5, 0, 0);
+        validateTable(document, 5, 5, 5, 0, 0);
+
+    }
+
+
+    @Test
+    public void testT3() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T3 S-Meto_Page29.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 6, 5, 0, 0);
+
+    }
+
+
+    @Test
+    public void testT4() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T4 138 IDD0000261736_Page16.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 5, 8, 1, 0);
+
+    }
+
+
+    @Test
+    public void testT5() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T5 VV-640252-Page16.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 6);
+        validateTable(document, 0, 1, 1, 0, 0);
+        validateTable(document, 1, 1, 1, 0, 0);
+        validateTable(document, 2, 1, 1, 0, 0);
+        validateTable(document, 3, 1, 1, 0, 0);
+        validateTable(document, 4, 1, 1, 0, 0);
+        validateTable(document, 5, 1, 1, 0, 0);
+
+    }
+
+
    @SneakyThrows
    private void toHtml(ClassificationDocument document, String filename) {

-        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
+        var tables = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList();
        StringBuilder sb = new StringBuilder();

        int currentPage = 1;
@@ -603,9 +783,19 @@ public class PdfSegmentationServiceTest extends AbstractTest {

    private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {

-        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        TablePageBlock table = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(tableIndex);
        List<List<Cell>> rows = table.getRows();
-        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
+        int emptyCellsFoundFound = rows.stream()
+                .flatMap(List::stream)
+                .toList()
+                .stream()
+                .filter(f -> f.toString().isEmpty())
+                .toList().size();

        for (List<Cell> row : table.getRows()) {
            row.forEach(r -> System.out.println(r.toString()));
@@ -620,11 +810,20 @@ public class PdfSegmentationServiceTest extends AbstractTest {

    private void validateTable(ClassificationDocument document, int tableIndex, List<List<String>> values) {

-        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        TablePageBlock table = document.getSections()
+                .stream()
+                .flatMap(paragraph -> paragraph.getTables()
+                        .stream())
+                .toList()
+                .get(tableIndex);
        List<List<Cell>> rows = table.getRows();

-        List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();
-        List<String> valuesFlattened = values.stream().flatMap(List::stream).toList();
+        List<Cell> rowsFlattened = rows.stream()
+                .flatMap(List::stream)
+                .toList();
+        List<String> valuesFlattened = values.stream()
+                .flatMap(List::stream)
+                .toList();

        for (int i = 0; i < valuesFlattened.size(); i++) {
            Cell cell = rowsFlattened.get(i);
@@ -637,7 +836,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {

    private void validateTableSize(ClassificationDocument document, int tableSize) {

-        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().size()).isEqualTo(tableSize);
+        assertThat(document.getSections()
+                           .stream()
+                           .flatMap(paragraph -> paragraph.getTables()
+                                   .stream())
+                           .toList().size()).isEqualTo(tableSize);

    }

@@ -21,15 +21,14 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Ta
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
-import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;

 import lombok.SneakyThrows;

@@ -57,9 +56,6 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
    @SneakyThrows
    public void testTableExtraction() {

-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
-        LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
-
        ClassPathResource resource = new ClassPathResource("files");
        List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
                .filter(path -> path.getFileName().toString().endsWith(".pdf"))
@@ -67,8 +63,8 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                .map(Path::toString)
                .toList();

-        for (int i = 0; i < pdfFileNames.size(); i++) {
-            writeJsons(Path.of(pdfFileNames.get(i)));
+        for (String pdfFileName : pdfFileNames) {
+            writeJsons(Path.of(pdfFileName));
        }
    }

@@ -80,21 +76,23 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                filename.toFile(),
                new ImageServiceResponse(),
                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
                filename.toFile().toString()));
        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
                filename.toFile(),
                new ImageServiceResponse(),
                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
                filename.toFile().toString()));
        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
-        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
-            String tmpFileNameBefore = "C:/Users/YANNIK~1/AppData/Local/Temp/before." + filename.getFileName().toString();
+        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure())) {
+            String tmpFileNameBefore = "/tmp/before." + filename.getFileName().toString();
            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
                PdfDraw.drawDocumentGraph(pdDocument, documentGraphBefore);
                pdDocument.save(tmpFileNameBefore);
            }
-            String tmpFileNameAfter = "C:/Users/YANNIK~1/AppData/Local/Temp/after." + filename.getFileName().toString();
+            String tmpFileNameAfter = "/tmp/after." + filename.getFileName().toString();
            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
                PdfDraw.drawDocumentGraph(pdDocument, documentGraphAfter);
                pdDocument.save(tmpFileNameAfter);
@@ -105,9 +103,9 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {


    @SneakyThrows
-    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2, String pdfName) {
+    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2) {

-        List listStructure1 = structure1.streamAllEntries()
+        List<Table> listStructure1 = structure1.streamAllEntries()
                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
                .map(DocumentStructure.EntryData::getProperties)
                .map(properties -> {
@@ -117,7 +115,7 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                })
                .toList();

-        List listStructure2 = structure2.streamAllEntries()
+        List<Table> listStructure2 = structure2.streamAllEntries()
                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
                .map(DocumentStructure.EntryData::getProperties)
                .map(properties -> {
@@ -128,8 +126,8 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                .toList();

        for (int i = 0; i < listStructure1.size(); i++) {
-            Table tableNode1 = (Table) listStructure1.get(i);
-            Table tableNode2 = (Table) listStructure2.get(i);
+            Table tableNode1 = listStructure1.get(i);
+            Table tableNode2 = listStructure2.get(i);
            if (tableNode1.getNumberOfRows() != tableNode2.getNumberOfRows() || tableNode1.getNumberOfCols() != tableNode2.getNumberOfCols()) {
                return false;
            }
@@ -20,6 +20,7 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.xmlunit.builder.Input;

 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.storage.commons.service.StorageService;
@@ -51,6 +52,8 @@ public abstract class AbstractTest {
    private RabbitTemplate rabbitTemplate;

    protected final static String ORIGIN_FILE_ID = "origin";
+
+    protected final static String VISUAL_LAYOUT_FILE = "visual";
    protected final static String TABLE_FILE_ID = "table";
    protected final static String IMAGE_FILE_ID = "image";
    protected final static String STRUCTURE_FILE_ID = "structure";
@@ -96,7 +99,7 @@ public abstract class AbstractTest {
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(String file) {

-        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json");
+        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json","visual_layout_parsing_response/empty.json");
    }


@@ -116,6 +119,7 @@ public abstract class AbstractTest {
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .visualLayoutParsingFileId(Optional.of(VISUAL_LAYOUT_FILE))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
                .positionBlockFileStorageId(POSITION_FILE_ID)
@@ -136,6 +140,16 @@ public abstract class AbstractTest {
        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream());
    }

+    @SneakyThrows
+    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile, String visualLayoutParsingResponseFile) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+        ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
+        ClassPathResource visualLayoutParsingResponseResource = new ClassPathResource(visualLayoutParsingResponseFile);
+
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream(), imageInfoFileResource.getInputStream(), visualLayoutParsingResponseResource.getInputStream());
+    }

    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream) {
@@ -147,6 +161,17 @@ public abstract class AbstractTest {
        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
    }

+    @SneakyThrows
+    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream, InputStream visualLayoutParsingResponseFileStream) {
+
+        storageService.storeObject(TenantContext.getTenantId(), IMAGE_FILE_ID, imageInfoStream);
+        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);
+        storageService.storeObject(TenantContext.getTenantId(),VISUAL_LAYOUT_FILE,visualLayoutParsingResponseFileStream );
+
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+    }
+

    @AfterEach
    public void cleanupStorage() {
@@ -10,6 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;

 import lombok.SneakyThrows;
@@ -25,7 +26,7 @@ public abstract class BuildDocumentTest extends AbstractTest {

        File fileResource = new ClassPathResource(filename).getFile();
        prepareStorage(filename);
-        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), filename);
+        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), new VisualLayoutParsingResponse(),filename);
    }


@@ -0,0 +1,8 @@
+{
+  "dossierId": "123",
+  "fileId": "123",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "EXTRACTED_TABLES.json.gz",
+  "data": [],
+  "X_TENANT_ID": ""
+}
@@ -12,6 +12,8 @@ public class ContentStreams {

    public static Identifier KNECON_LAYOUT = new Identifier("Layout grid", COSName.getPDFName("KNECON_LAYOUT"), true);

+    public static Identifier KNECON_VISUAL_PARSING = new Identifier("Layout grid - visual", COSName.getPDFName("KNECON_VISUAL_PARSING"), true);
+
    public static Identifier KNECON_OCR = new Identifier("OCR", COSName.getPDFName("KNECON_OCR"), false);

    public static Identifier KNECON_OCR_TEXT_DEBUG = new Identifier("OCR Text", COSName.getPDFName("KNECON_OCR_TEXT_DEBUG"), true);
@@ -24,7 +26,14 @@ public class ContentStreams {

    public static Identifier ESCAPE_END = new Identifier("escape start", COSName.getPDFName("ESCAPE_END"), false);

-    public static List<Identifier> allContentStreams = List.of(KNECON_LAYOUT, KNECON_OCR, KNECON_OCR_BBOX_DEBUG, KNECON_OCR_TEXT_DEBUG, OTHER, ESCAPE_START, ESCAPE_END);
+    public static List<Identifier> allContentStreams = List.of(KNECON_LAYOUT,
+                                                               KNECON_VISUAL_PARSING,
+                                                               KNECON_OCR,
+                                                               KNECON_OCR_BBOX_DEBUG,
+                                                               KNECON_OCR_TEXT_DEBUG,
+                                                               OTHER,
+                                                               ESCAPE_START,
+                                                               ESCAPE_END);

    public record Identifier(String name, COSName cosName, boolean optionalContent) {
Author	SHA1	Message	Date
Kilian Schüttler	e7bf607663	Merge branch 'hotfix' into 'main' hotfix: double viewerdoc writes in rare cases lead to some contentstreams not being written See merge request fforesight/layout-parser!104	2024-02-26 12:32:14 +01:00
Kilian Schuettler	f4d789311c	hotfix: double viewerdoc writes in rare cases lead to some contentstreams not being written	2024-02-26 12:24:15 +01:00
Yannik Hampe	9817eae897	Merge branch 'RED-8481-fix' into 'main' Red 8481 fix See merge request fforesight/layout-parser!103	2024-02-23 14:08:57 +01:00
yhampe	477f6af886	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results checkstyle	2024-02-23 14:02:53 +01:00
yhampe	2c171b6a9e	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results codestyle	2024-02-23 13:55:11 +01:00
yhampe	71477dabde	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results codestyle	2024-02-23 12:46:51 +01:00
yhampe	a927cbd9dc	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results fixed tests	2024-02-23 12:38:05 +01:00
yhampe	a1521877d7	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results added a source label to image properties to enable rules	2024-02-23 12:20:11 +01:00
Maverick Studer	f4b6386e1c	Merge branch 'RED-8550-testing' into 'main' RED-8550: Faulty table recognition and text duplication leads to huge sections See merge request fforesight/layout-parser!102	2024-02-21 13:54:30 +01:00
Maverick Studer	1d64028158	RED-8550: Faulty table recognition and text duplication leads to huge sections	2024-02-21 13:54:30 +01:00
Yannik Hampe	0979a267d4	Merge branch 'RED-8481' into 'main' RED-8481: Use visual layout parsing to detect signatures See merge request fforesight/layout-parser!101	2024-02-15 13:07:06 +01:00
yhampe	cc77d19500	RED-8481: Use visual layout parsing to detect signatures addressed review comments	2024-02-15 13:01:30 +01:00
yhampe	fa048b2fe0	RED-8481: Use visual layout parsing to detect signatures addressed review comments	2024-02-15 12:19:26 +01:00
yhampe	bdf1161c91	RED-8481: Use visual layout parsing to detect signatures addressed review comments	2024-02-15 12:12:23 +01:00
yhampe	b4a225144d	RED-8481: Use visual layout parsing to detect signatures working on failing tests	2024-02-15 10:16:07 +01:00
yhampe	903b1c1fd4	RED-8481: Use visual layout parsing to detect signatures fixed failing tests because of null pointer	2024-02-15 09:27:07 +01:00
yhampe	c3e7582ee3	RED-8481: Use visual layout parsing to detect signatures fixed failing tests because of null pointer	2024-02-14 12:33:36 +01:00
yhampe	cfc5db45cd	RED-8481: Use visual layout parsing to detect signatures fixed failing tests because of null pointer	2024-02-14 12:24:32 +01:00
yhampe	fbd0196719	RED-8481: Use visual layout parsing to detect signatures implemented visuallayoutparsingresult	2024-02-14 12:16:37 +01:00