RED-7141: Implemented docstrum layout parsing

Merge branch 'RED-8481-hotfix' into 'main'
RED-8481: Use visual layout parsing to detect signatures See merge request fforesight/layout-parser!106
2024-03-06 11:18:40 +01:00 · 2024-02-29 09:39:17 +01:00 · 2024-02-29 09:22:41 +01:00 · 2024-02-29 09:22:27 +01:00 · 2024-02-29 09:21:47 +01:00 · 2024-02-28 16:13:56 +01:00
114 changed files with 2541 additions and 2737 deletions
@@ -5,6 +5,7 @@ import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Stream;

 import io.swagger.v3.oas.annotations.media.Schema;
@@ -54,6 +55,13 @@ public class DocumentStructure implements Serializable {

    }

+    @Schema(description = "Object containing the extra field names, a duplicate paragraph has in its properties field.")
+    public static class DuplicateParagraphProperties implements Serializable {
+
+        public static final String UNSORTED_TEXTBLOCK_ID = "utbid";
+
+    }
+
    public static final String RECTANGLE_DELIMITER = ";";


@@ -115,6 +123,8 @@ public class DocumentStructure implements Serializable {
        Map<String, String> properties;
        @Schema(description = "All child Entries of this Entry.", example = "[1, 2, 3]")
        List<EntryData> children;
+        @Schema(description = "Describes the origin of the semantic node", example = "[ALGORITHM]")
+        Set<LayoutEngine> engines;


        @Override
@@ -0,0 +1,6 @@
+package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
+
+public enum LayoutEngine {
+    ALGORITHM,
+    AI
+}
@@ -18,11 +18,15 @@ public record LayoutParsingRequest(

        @Schema(description = "Path to the original PDF file.")//
        @NonNull String originFileStorageId,//
+
+
        @Schema(description = "Optional Path to the table extraction file.")//
        Optional<String> tablesFileStorageId,//
        @Schema(description = "Optional Path to the image classification file.")//
        Optional<String> imagesFileStorageId,//

+        @Schema(description = "Optional Path to the the visual layout parsing service file") Optional<String> visualLayoutParsingFileId,//
+
        @Schema(description = "Path where the Document Structure File will be stored.")//
        @NonNull String structureFileStorageId,//
        @Schema(description = "Path where the Research Data File will be stored.")//
@@ -2,6 +2,9 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;

 public enum LayoutParsingType {
    REDACT_MANAGER,
-    TAAS,
-    DOCUMINE
+    REDACT_MANAGER_OLD,
+    REDACT_MANAGER_PARAGRAPH_DEBUG,
+    DOCUMINE,
+    CLARIFYND,
+    CLARIFYND_PARAGRAPH_DEBUG
 }
@@ -24,4 +24,5 @@ dependencies {
    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
    implementation("org.springframework.boot:spring-boot-starter-web:3.1.3")
+    implementation("org.jgrapht:jgrapht-core:1.5.2")
 }
@@ -10,7 +10,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;

 import org.apache.pdfbox.Loader;
@@ -27,29 +26,29 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.VisualLayoutParsingAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
@@ -77,19 +76,18 @@ public class LayoutParsingPipeline {
    CvTableParsingAdapter cvTableParsingAdapter;
    LayoutParsingStorageService layoutParsingStorageService;
    SectionsBuilderService sectionsBuilderService;
-    TaasClassificationService taasClassificationService;
    RedactManagerClassificationService redactManagerClassificationService;
    DocuMineClassificationService docuMineClassificationService;
    SimplifiedSectionTextService simplifiedSectionTextService;
    BodyTextFrameService bodyTextFrameService;
    RulingCleaningService rulingCleaningService;
    TableExtractionService tableExtractionService;
-    TaasBlockificationService taasBlockificationService;
    DocuMineBlockificationService docuMineBlockificationService;
    RedactManagerBlockificationService redactManagerBlockificationService;
+    DocstrumBlockificationService docstrumBlockificationService;
    LayoutGridService layoutGridService;
    ObservationRegistry observationRegistry;
-    DocstrumSegmentationService docstrumSegmentationService;
+    VisualLayoutParsingAdapter visualLayoutParsingAdapter;


    public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -100,6 +98,11 @@ public class LayoutParsingPipeline {
        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);

+        VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse();
+        if (layoutParsingRequest.visualLayoutParsingFileId().isPresent()) {
+            visualLayoutParsingResponse = layoutParsingStorageService.getVisualLayoutParsingFile(layoutParsingRequest.visualLayoutParsingFileId().get());
+        }
+
        ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
        if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
            imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
@@ -114,11 +117,12 @@ public class LayoutParsingPipeline {
                originFile,
                imageServiceResponse,
                tableServiceResponse,
+                visualLayoutParsingResponse,
                layoutParsingRequest.identifier().toString());

        log.info("Building document graph for {}", layoutParsingRequest.identifier());

-        Document documentGraph = observeBuildDocumentGraph(classificationDocument);
+        Document documentGraph = observeBuildDocumentGraph(layoutParsingRequest.layoutParsingType(), classificationDocument);

        log.info("Creating viewer document for {}", layoutParsingRequest.identifier());

@@ -130,7 +134,7 @@ public class LayoutParsingPipeline {
        layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
        layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);

-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.CLARIFYND)) {
            log.info("Building research document data for {}", layoutParsingRequest.identifier());
            var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
            layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
@@ -170,13 +174,13 @@ public class LayoutParsingPipeline {
    }


-    private Document observeBuildDocumentGraph(ClassificationDocument classificationDocument) {
+    private Document observeBuildDocumentGraph(LayoutParsingType layoutParsingType, ClassificationDocument classificationDocument) {

        AtomicReference<Document> documentReference = new AtomicReference<>();

-        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry).contextualName("build-document-graph").observe(() -> {
-            documentReference.set(DocumentGraphFactory.buildDocumentGraph(classificationDocument));
-        });
+        Observation.createNotStarted("LayoutParsingPipeline", observationRegistry)
+                .contextualName("build-document-graph")
+                .observe(() -> documentReference.set(DocumentGraphFactory.buildDocumentGraph(layoutParsingType, classificationDocument)));

        return documentReference.get();
    }
@@ -202,13 +206,14 @@ public class LayoutParsingPipeline {
                                              File originFile,
                                              ImageServiceResponse imageServiceResponse,
                                              TableServiceResponse tableServiceResponse,
+                                              VisualLayoutParsingResponse visualLayoutParsingResponse,
                                              String identifier) {

        PDDocument originDocument = openDocument(originFile);
        addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
        Map<Integer, List<TableCells>> pdfTableCells = cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse);
        Map<Integer, List<ClassifiedImage>> pdfImages = imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse);
-
+        Map<Integer, List<ClassifiedImage>> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse);
        ClassificationDocument classificationDocument = new ClassificationDocument();
        List<ClassificationPage> classificationPages = new ArrayList<>();

@@ -247,36 +252,15 @@ public class LayoutParsingPipeline {
            PDRectangle cropbox = pdPage.getCropBox();
            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());

-            // Docstrum
-            AtomicInteger num = new AtomicInteger(pageNumber);
-            var zones = docstrumSegmentationService.segmentPage(stripper.getTextPositionSequences());
+            List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());

-            List<AbstractPageBlock> pageBlocks = new ArrayList<>();
-            AtomicInteger numOnPage = new AtomicInteger(1);
-//            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-            zones.forEach(zone -> {
-
-                List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-                zone.getLines().forEach(line -> {
-                    line.getWords().forEach(word -> {
-                        textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), num.get()));
-                    });
-                });
-
-                var cps = redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-                cps.getTextBlocks().forEach(cp -> {
-                    pageBlocks.add(redactManagerBlockificationService.buildTextBlock(((TextPageBlock) cp).getSequences(), numOnPage.getAndIncrement()));
-                });
-            });
-
-//            ClassificationPage classificationPage = switch (layoutParsingType) {
-//                case REDACT_MANAGER -> redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//                case TAAS -> taasBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//                case DOCUMINE -> docuMineBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical());
-//            };
-
-            ClassificationPage classificationPage = new ClassificationPage(pageBlocks);
+            ClassificationPage classificationPage = switch (layoutParsingType) {
+                case REDACT_MANAGER_OLD ->
+                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+                case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, true);
+                case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), emptyTableCells, false);
+            };

            classificationPage.setCleanRulings(cleanRulings);
            classificationPage.setRotation(rotation);
@@ -294,7 +278,21 @@ public class LayoutParsingPipeline {
                imageServiceResponseAdapter.findOcr(classificationPage);
            }

-            tableExtractionService.extractTables(cleanRulings, classificationPage);
+            if (signatures.containsKey(pageNumber)) {
+                if (classificationPage.getImages() == null || classificationPage.getImages().size() == 0) {
+                    classificationPage.setImages(signatures.get(pageNumber));
+                } else {
+                    classificationPage.getImages().addAll(signatures.get(pageNumber));
+                }
+            }
+
+            tableExtractionService.extractTables(emptyTableCells, classificationPage);
+
+            if (layoutParsingType == LayoutParsingType.REDACT_MANAGER) {
+                docstrumBlockificationService.combineBlocks(classificationPage);
+            } else if (layoutParsingType == LayoutParsingType.CLARIFYND) {
+                docstrumBlockificationService.mergeZones(classificationPage.getTextBlocks());
+            }

            buildPageStatistics(classificationPage);
            increaseDocumentStatistics(classificationPage, classificationDocument);
@@ -308,24 +306,21 @@ public class LayoutParsingPipeline {
        bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
        log.info("Classify TextBlocks for {}", identifier);
        switch (layoutParsingType) {
-            case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG ->
+                    redactManagerClassificationService.classifyDocument(classificationDocument);
            case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
-            case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
        }

-        List<ClassificationSection> sections = new ArrayList<>();
-        for (var page : classificationPages) {
-            page.getTextBlocks().forEach(block -> {
-                block.setPage(page.getPageNumber());
-                var section = sectionsBuilderService.buildTextBlock(List.of(block), "a");
-                sections.add(section);
-            });
-        }
-        classificationDocument.setSections(sections);
-
        log.info("Building Sections for {}", identifier);
-//        sectionsBuilderService.buildSections(classificationDocument);
-//        sectionsBuilderService.addImagesToSections(classificationDocument);
+
+        switch (layoutParsingType) {
+            case CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_PARAGRAPH_DEBUG -> sectionsBuilderService.buildParagraphDebugSections(classificationDocument);
+            default -> {
+                sectionsBuilderService.buildSections(classificationDocument);
+                sectionsBuilderService.addImagesToSections(classificationDocument);
+            }
+        }
+
        return classificationDocument;
    }

@@ -359,9 +354,7 @@ public class LayoutParsingPipeline {

    private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {

-        if (!classificationPage.isLandscape()) {
-            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-        }
+        document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
        document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
        document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
        document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());
@@ -10,8 +10,6 @@ import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.Optional;

-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -22,6 +20,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.Researc
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;

 import io.micrometer.observation.annotation.Observed;
@@ -79,6 +78,14 @@ public class LayoutParsingStorageService {
        }
    }

+    public VisualLayoutParsingResponse getVisualLayoutParsingFile(String storageId) throws IOException {
+
+        try (InputStream inputStream = getObject(storageId)) {
+            VisualLayoutParsingResponse visualLayoutParsingResponse = objectMapper.readValue(inputStream, VisualLayoutParsingResponse.class);
+            return visualLayoutParsingResponse;
+        }
+    }
+

    @Observed(name = "LayoutParsingStorageService", contextualName = "store-document-data")
    public void storeDocumentData(LayoutParsingRequest layoutParsingRequest, DocumentData documentData) {
@@ -6,12 +6,14 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageB

 import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;

@Data
@AllArgsConstructor
@NoArgsConstructor
-public abstract class AbstractPageBlock {
+@EqualsAndHashCode(callSuper = true)
+public abstract class AbstractPageBlock extends Rectangle {

    @JsonIgnore
    protected float minX;
@@ -94,7 +96,7 @@ public abstract class AbstractPageBlock {

        return this.minX - threshold <= apb.getMaxX() && this.maxX + threshold >= apb.getMinX();
    }
-
+    

    public abstract boolean isEmpty();

@@ -12,16 +12,15 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRul
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;

 import lombok.Data;
-import lombok.Getter;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
+import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;

@Data
@RequiredArgsConstructor
 public class ClassificationPage {

    @NonNull
-    @Getter
    private List<AbstractPageBlock> textBlocks;

    private List<ClassifiedImage> images = new ArrayList<>();
@@ -10,11 +10,11 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -30,6 +30,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Document implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
    Set<Page> pages;
    DocumentTree documentTree;
    Integer numberOfPages;
@@ -48,7 +51,7 @@ public class Document implements GenericSemanticNode {
    public TextBlock getTextBlock() {

        if (textBlock == null) {
-            textBlock = streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
+            textBlock = GenericSemanticNode.super.getTextBlock();
        }
        return textBlock;
    }
@@ -56,13 +59,14 @@ public class Document implements GenericSemanticNode {

    public List<Section> getMainSections() {

-        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node).collect(Collectors.toList());
+        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node)
+                .collect(Collectors.toList());
    }


    public Stream<TextBlock> streamTerminalTextBlocksInOrder() {

-        return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock);
+        return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getTextBlock);
    }


@@ -83,13 +87,16 @@ public class Document implements GenericSemanticNode {
    @Override
    public Headline getHeadline() {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElse(Headline.builder().build());
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node)
+                .findFirst()
+                .orElse(Headline.builder().build());
    }


    private Stream<SemanticNode> streamAllNodes() {

-        return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode);
+        return documentTree.allEntriesInOrder()
+                .map(DocumentTree.Entry::getNode);
    }


@@ -0,0 +1,34 @@
+package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
+
+import java.util.stream.Stream;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+@SuperBuilder
+public class DuplicatedParagraph extends Paragraph {
+
+    TextBlock unsortedLeafTextBlock;
+
+
+    @Override
+    public TextBlock getTextBlock() {
+
+        return Stream.of(leafTextBlock, unsortedLeafTextBlock).collect(new TextBlockCollector());
+
+    }
+
+
+    @Override
+    public String toString() {
+
+        return super.toString();
+    }
+
+}
@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -26,6 +27,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Footer implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
    List<Integer> treeId;
    TextBlock leafTextBlock;

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Header implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;
    TextBlock leafTextBlock;

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Headline implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;
    TextBlock leafTextBlock;

@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -29,6 +30,9 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Image implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
    List<Integer> treeId;
    String id;

@@ -66,7 +70,9 @@ public class Image implements GenericSemanticNode {
    @Override
    public TextBlock getTextBlock() {

-        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
    }


@@ -6,6 +6,8 @@ public enum ImageType {
    LOGO,
    FORMULA,
    SIGNATURE,
+
+    SIGNATURE_VISUAL,
    OTHER,
    OCR;

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -17,13 +18,17 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;

@Data
-@Builder
+@SuperBuilder
@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
+@FieldDefaults(level = AccessLevel.PROTECTED)
 public class Paragraph implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
    List<Integer> treeId;
    TextBlock leafTextBlock;

@@ -6,11 +6,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +27,8 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Section implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;

    TextBlock textBlock;
@@ -50,7 +52,8 @@ public class Section implements GenericSemanticNode {

    public boolean hasTables() {

-        return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent();
+        return streamAllSubNodesOfType(NodeType.TABLE).findAny()
+                .isPresent();
    }


@@ -58,7 +61,7 @@ public class Section implements GenericSemanticNode {
    public TextBlock getTextBlock() {

        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = GenericSemanticNode.super.getTextBlock();
        }
        return textBlock;
    }
@@ -12,6 +12,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
@@ -19,6 +20,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.E
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 public interface SemanticNode {
@@ -38,7 +40,10 @@ public interface SemanticNode {
     *
     * @return TextBlock containing all AtomicTextBlocks that are located under this Node.
     */
-    TextBlock getTextBlock();
+    default TextBlock getTextBlock() {
+
+        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getTextBlock).collect(new TextBlockCollector());
+    }


    /**
@@ -335,6 +340,23 @@ public interface SemanticNode {
    }


+    /**
+     * returns the set of layoutengines.
+     *
+     * @return set of layoutengines.
+     */
+    Set<LayoutEngine> getEngines();
+
+
+    /**
+     * adds a layoutengine to the set.
+     */
+    default void addEngine(LayoutEngine engine) {
+
+        getEngines().add(engine);
+    }
+
+
    /**
     * Streams all children located directly underneath this node in the DocumentTree.
     *
@@ -426,6 +448,7 @@ public interface SemanticNode {
    /**
     * TODO: this produces unwanted results for sections spanning multiple columns.
     * Computes the Union of the bounding boxes of all children recursively.
+     *
     * @return The union of the BoundingBoxes of all children
     */
    private Map<Page, Rectangle2D> getBBoxFromChildren() {
@@ -12,6 +12,7 @@ import java.util.Set;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -31,6 +32,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class Table implements SemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;
    DocumentTree documentTree;

@@ -53,8 +56,7 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {

-        return IntStream.range(0, numberOfRows)
-                .boxed()
+        return IntStream.range(0, numberOfRows).boxed()
                .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                .flatMap(this::streamRow)
                .map(TableCell::getEntities)
@@ -71,8 +73,11 @@ public class Table implements SemanticNode {
     */
    public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {

-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
-        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
+        String rowText = streamRow(row).map(TableCell::getTextBlock)
+                .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
+        return strings.stream()
+                .map(String::toLowerCase)
+                .allMatch(rowText::contains);
    }


@@ -85,9 +90,13 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {

-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
        return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
-                .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -100,9 +109,13 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {

-        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
        return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -114,12 +127,15 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {

-        List<Integer> rowsWithEntityOfType = IntStream.range(0, numberOfRows)
-                .boxed()
-                .filter(rowNumber -> streamEntityTypesInRow(rowNumber).anyMatch(existingType -> types.stream().anyMatch(typeToCheck -> typeToCheck.equals(existingType))))
+        List<Integer> rowsWithEntityOfType = IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamEntityTypesInRow(rowNumber).anyMatch(existingType -> types.stream()
+                        .anyMatch(typeToCheck -> typeToCheck.equals(existingType))))
                .toList();

-        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
+        return rowsWithEntityOfType.stream()
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


@@ -131,18 +147,24 @@ public class Table implements SemanticNode {
     */
    public Stream<RedactionEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {

-        List<Integer> rowsWithNoEntityOfType = IntStream.range(0, numberOfRows)
-                .boxed()
-                .filter(rowNumber -> streamEntityTypesInRow(rowNumber).noneMatch(existingType -> types.stream().anyMatch(typeToCheck -> typeToCheck.equals(existingType))))
+        List<Integer> rowsWithNoEntityOfType = IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamEntityTypesInRow(rowNumber).noneMatch(existingType -> types.stream()
+                        .anyMatch(typeToCheck -> typeToCheck.equals(existingType))))
                .toList();

-        return rowsWithNoEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
+        return rowsWithNoEntityOfType.stream()
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
    }


    private Stream<String> streamEntityTypesInRow(Integer rowNumber) {

-        return streamRow(rowNumber).map(TableCell::getEntities).flatMap(Collection::stream).map(RedactionEntity::getType).distinct();
+        return streamRow(rowNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .map(RedactionEntity::getType)
+                .distinct();
    }


@@ -159,7 +181,8 @@ public class Table implements SemanticNode {
            throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
        }
        int idx = row * numberOfCols + col;
-        return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
+        return (TableCell) documentTree.getEntryById(treeId).getChildren()
+                .get(idx).getNode();
    }


@@ -196,7 +219,8 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamCol(int col) {

-        return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
+        return IntStream.range(0, numberOfRows).boxed()
+                .map(row -> getCell(row, col));
    }


@@ -208,9 +232,11 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamRow(int row) {

-        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
+        return IntStream.range(0, numberOfCols).boxed()
+                .map(col -> getCell(row, col));
    }

+
    /**
     * Streams all TableCells row-wise and filters them with header == true.
     *
@@ -231,7 +257,8 @@ public class Table implements SemanticNode {
     */
    public Stream<TableCell> streamHeadersForCell(int row, int col) {

-        return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
+        return Stream.concat(streamRow(row), streamCol(col))
+                .filter(TableCell::isHeader);
    }


@@ -304,7 +331,7 @@ public class Table implements SemanticNode {
    public TextBlock getTextBlock() {

        if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = SemanticNode.super.getTextBlock();
        }
        return textBlock;
    }
@@ -315,6 +342,8 @@ public class Table implements SemanticNode {

        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
    }
+
+
    @Override
    public Map<Page, Rectangle2D> getBBox() {

@@ -323,4 +352,5 @@ public class Table implements SemanticNode {
        }
        return bBoxCache;
    }
+
 }
@@ -7,6 +7,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
@@ -26,6 +27,8 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class TableCell implements GenericSemanticNode {

+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
    List<Integer> treeId;
    int row;
    int col;
@@ -84,14 +87,16 @@ public class TableCell implements GenericSemanticNode {

    private TextBlock buildTextBlock() {

-        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
    }


    @Override
    public String toString() {

-        return treeId + ": " + NodeType.TABLE_CELL + ": " + this.buildTextBlock().buildSummary();
+        return treeId + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
    }

 }
@@ -4,18 +4,21 @@ import java.awt.geom.Rectangle2D;

 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;

+import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;

@Data
@RequiredArgsConstructor
+@AllArgsConstructor
 public class ClassifiedImage {

    @NonNull
    private Rectangle2D position;
    @NonNull
    private ImageType imageType;
+    private boolean sourceByAi;
    private boolean isAppendedToSection;
    @NonNull
    private boolean hasTransparency;
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -36,6 +37,12 @@ public class Cell extends Rectangle {
    }


+    public Cell(Rectangle2D r) {
+
+        super((float) r.getY(), (float) r.getX(), (float) r.getWidth(), (float) r.getHeight());
+    }
+
+
    public void addTextBlock(TextPageBlock textBlock) {

        textBlocks.add(textBlock);
@@ -20,7 +20,8 @@ import lombok.extern.slf4j.Slf4j;
@SuppressWarnings("all")
 public class Ruling extends Line2D.Float {

-    private static int PERPENDICULAR_PIXEL_EXPAND_AMOUNT = 2;
+    public static final int PERPENDICULAR_UNIT_EXPAND_AMOUNT = 2;
+    public static final int COLINEAR_OR_PARALLEL_UNIT_EXPAND_AMOUNT = 2;


    public Ruling(Point2D p1, Point2D p2) {
@@ -110,8 +111,8 @@ public class Ruling extends Line2D.Float {
        });

        for (Ruling h : horizontals) {
-            sos.add(new SortObject(SOType.HLEFT, h.getLeft() - PERPENDICULAR_PIXEL_EXPAND_AMOUNT, h));
-            sos.add(new SortObject(SOType.HRIGHT, h.getRight() + PERPENDICULAR_PIXEL_EXPAND_AMOUNT, h));
+            sos.add(new SortObject(SOType.HLEFT, h.getLeft() - PERPENDICULAR_UNIT_EXPAND_AMOUNT, h));
+            sos.add(new SortObject(SOType.HRIGHT, h.getRight() + PERPENDICULAR_UNIT_EXPAND_AMOUNT, h));
        }

        for (Ruling v : verticals) {
@@ -151,7 +152,7 @@ public class Ruling extends Line2D.Float {
                            if (i == null) {
                                continue;
                            }
-                            rv.put(i, new Ruling[]{h.getKey().expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT), so.ruling.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)});
+                            rv.put(i, new Ruling[]{h.getKey().expand(PERPENDICULAR_UNIT_EXPAND_AMOUNT), so.ruling.expand(PERPENDICULAR_UNIT_EXPAND_AMOUNT)});
                        } catch (UnsupportedOperationException e) {
                            log.info("Some line are oblique, ignoring...");
                            continue;
@@ -267,7 +268,7 @@ public class Ruling extends Line2D.Float {
    }


-    public boolean nearlyIntersects(Ruling another, int colinearOrParallelExpandAmount) {
+    public boolean nearlyIntersects(Ruling another) {

        if (this.intersectsLine(another)) {
            return true;
@@ -276,9 +277,9 @@ public class Ruling extends Line2D.Float {
        boolean rv = false;

        if (this.perpendicularTo(another)) {
-            rv = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT).intersectsLine(another);
+            rv = this.expand(PERPENDICULAR_UNIT_EXPAND_AMOUNT).intersectsLine(another);
        } else {
-            rv = this.expand(colinearOrParallelExpandAmount).intersectsLine(another.expand(colinearOrParallelExpandAmount));
+            rv = this.expand(COLINEAR_OR_PARALLEL_UNIT_EXPAND_AMOUNT).intersectsLine(another.expand(COLINEAR_OR_PARALLEL_UNIT_EXPAND_AMOUNT));
        }

        return rv;
@@ -319,8 +320,8 @@ public class Ruling extends Line2D.Float {

    public Point2D intersectionPoint(Ruling other) {

-        Ruling this_l = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
-        Ruling other_l = other.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
+        Ruling this_l = this.expand(PERPENDICULAR_UNIT_EXPAND_AMOUNT);
+        Ruling other_l = other.expand(PERPENDICULAR_UNIT_EXPAND_AMOUNT);
        Ruling horizontal, vertical;

        if (!this_l.intersectsLine(other_l)) {
@@ -1,18 +1,18 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;

 import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

 import lombok.Getter;
 import lombok.Setter;
@@ -21,7 +21,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
 public class TablePageBlock extends AbstractPageBlock {

-    private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();
+    public static final double CELL_AREA_CONTAINED_THRESHOLD = 0.98;
+    private final TreeMap<CellPosition, Cell> cellTreeMap = new TreeMap<>();

    private final int rotation;
    @Getter
@@ -30,10 +31,14 @@ public class TablePageBlock extends AbstractPageBlock {
    private int unrotatedRowCount;
    private int unrotatedColCount;
    private List<List<Cell>> rows;
+    @Getter
+    @Setter
+    private List<Cell> cells;


    public TablePageBlock(List<Cell> cells, Rectangle area, int rotation) {

+        this.cells = cells;
        addCells(cells);
        minX = area.getLeft();
        minY = area.getBottom();
@@ -50,6 +55,7 @@ public class TablePageBlock extends AbstractPageBlock {
        return getColCount() == 0 || getRowCount() == 0;
    }

+
    public List<List<Cell>> getRows() {

        if (rows == null) {
@@ -80,14 +86,17 @@ public class TablePageBlock extends AbstractPageBlock {

    public int getColCount() {

-        return getRows().stream().mapToInt(List::size).max().orElse(0);
+        return getRows().stream()
+                .mapToInt(List::size)
+                .max()
+                .orElse(0);

    }


    /**
     * Detect header cells (either first row or first column):
-     * Column is marked as header if cell text is bold and row cell text is not bold.
+     * Column is marked as header if originalCell text is bold and row originalCell text is not bold.
     * Defaults to row.
     */
    private void computeHeaders() {
@@ -95,7 +104,7 @@ public class TablePageBlock extends AbstractPageBlock {
        if (rows == null) {
            rows = computeRows();
        }
-        // A bold cell is a header cell as long as every cell to the left/top is bold, too
+        // A bold originalCell is a header originalCell as long as every originalCell to the left/top is bold, too
        // we move from left to right and top to bottom
        for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
            List<Cell> rowCells = rows.get(rowIndex);
@@ -120,7 +129,8 @@ public class TablePageBlock extends AbstractPageBlock {
                List<Cell> cellsToTheTop = new ArrayList<>();
                for (int i = 0; i < rowIndex; i++) {
                    try {
-                        cellsToTheTop.add(rows.get(i).get(colIndex));
+                        cellsToTheTop.add(rows.get(i)
+                                                  .get(colIndex));
                    } catch (IndexOutOfBoundsException e) {
                        log.debug("No cell {} in row {}, ignoring.", colIndex, rowIndex);
                    }
@@ -135,7 +145,8 @@ public class TablePageBlock extends AbstractPageBlock {
                if (lastHeaderCell != null) {
                    cell.getHeaderCells().add(lastHeaderCell);
                }
-                if (!cell.getTextBlocks().isEmpty() && cell.getTextBlocks().get(0).getMostPopularWordStyle().equals("bold")) {
+                if (!cell.getTextBlocks().isEmpty() && cell.getTextBlocks()
+                        .get(0).getMostPopularWordStyle().equals("bold")) {
                    cell.setHeaderCell(true);
                }
            }
@@ -151,7 +162,7 @@ public class TablePageBlock extends AbstractPageBlock {
            for (int i = 0; i < unrotatedColCount; i++) { // rows
                List<Cell> lastRow = new ArrayList<>();
                for (int j = unrotatedRowCount - 1; j >= 0; j--) { // cols
-                    Cell cell = cells.get(new CellPosition(j, i));
+                    Cell cell = cellTreeMap.get(new CellPosition(j, i));
                    if (cell != null) {
                        lastRow.add(cell);
                    }
@@ -162,7 +173,7 @@ public class TablePageBlock extends AbstractPageBlock {
            for (int i = unrotatedColCount - 1; i >= 0; i--) { // rows
                List<Cell> lastRow = new ArrayList<>();
                for (int j = 0; j < unrotatedRowCount; j++) { // cols
-                    Cell cell = cells.get(new CellPosition(j, i));
+                    Cell cell = cellTreeMap.get(new CellPosition(j, i));
                    if (cell != null) {
                        lastRow.add(cell);
                    }
@@ -173,7 +184,7 @@ public class TablePageBlock extends AbstractPageBlock {
            for (int i = 0; i < unrotatedRowCount; i++) {
                List<Cell> lastRow = new ArrayList<>();
                for (int j = 0; j < unrotatedColCount; j++) {
-                    Cell cell = cells.get(new CellPosition(i, j)); // JAVA_8 use getOrDefault()
+                    Cell cell = cellTreeMap.get(new CellPosition(i, j)); // JAVA_8 use getOrDefault()
                    if (cell != null) {
                        lastRow.add(cell);
                    }
@@ -187,17 +198,6 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-    private void add(Cell chunk, int row, int col) {
-
-        unrotatedRowCount = Math.max(unrotatedRowCount, row + 1);
-        unrotatedColCount = Math.max(unrotatedColCount, col + 1);
-
-        CellPosition cp = new CellPosition(row, col);
-        cells.put(cp, chunk);
-
-    }
-
-
    private void addCells(List<Cell> cells) {

        if (cells.isEmpty()) {
@@ -206,11 +206,12 @@ public class TablePageBlock extends AbstractPageBlock {

        cells.removeIf(cell -> cell.getWidth() < 1.1 || cell.getHeight() < 1.1);

-        List<List<Cell>> rowsOfCells = calculateStructure(cells);
+        List<List<Cell>> rowsOfCellsMatrix = calculateTableStructure(cells);

-        for (int i = 0; i < rowsOfCells.size(); i++) {
-            for (int j = 0; j < rowsOfCells.get(i).size(); j++) {
-                add(rowsOfCells.get(i).get(j), i, j);
+        for (int i = 0; i < rowsOfCellsMatrix.size(); i++) {
+            for (int j = 0; j < rowsOfCellsMatrix.get(i).size(); j++) {
+                addCellToRowAndCol(rowsOfCellsMatrix.get(i)
+                                           .get(j), i, j);
            }
        }

@@ -221,29 +222,36 @@ public class TablePageBlock extends AbstractPageBlock {
     * Calculates the structure of the table. For spanning rows and columns multiple cells with the same values will be inserted.
     *
     * @param cells The found cells
-     * @return TablePageBlock Structure
+     * @return TablePageBlock Structure as a rows of cells matrix
     */
-    private List<List<Cell>> calculateStructure(List<Cell> cells) {
-
-        List<List<Cell>> matrix = new ArrayList<>();
+    private List<List<Cell>> calculateTableStructure(List<Cell> cells) {

        if (cells.isEmpty()) {
-            return matrix;
+            return new ArrayList<>();
        }

        Set<Float> uniqueX = new HashSet<>();
        Set<Float> uniqueY = new HashSet<>();
-        cells.stream().filter(c -> !c.getTextBlocks().isEmpty() || c.getHeight() > 3 && c.getWidth() > 3).forEach(c -> {
-            uniqueX.add(c.getLeft());
-            uniqueX.add(c.getRight());
-            uniqueY.add(c.getBottom());
-            uniqueY.add(c.getTop());
-        });
+        cells.stream()
+                .filter(c -> !c.getTextBlocks().isEmpty() || c.getHeight() > 3 && c.getWidth() > 3)
+                .forEach(c -> {
+                    uniqueX.add(c.getLeft());
+                    uniqueX.add(c.getRight());
+                    uniqueY.add(c.getBottom());
+                    uniqueY.add(c.getTop());
+                });

-        var sortedUniqueX = uniqueX.stream().sorted().toList();
-        var sortedUniqueY = uniqueY.stream().sorted().toList();
+        var sortedUniqueX = uniqueX.stream()
+                .sorted()
+                .toList();
+        var sortedUniqueY = uniqueY.stream()
+                .sorted()
+                .toList();
+
+        List<List<Cell>> rowsOfCells = new ArrayList<>();

        Float prevY = null;
+
        for (Float y : sortedUniqueY) {

            List<Cell> row = new ArrayList<>();
@@ -252,44 +260,87 @@ public class TablePageBlock extends AbstractPageBlock {
            for (Float x : sortedUniqueX) {

                if (prevY != null && prevX != null) {
-                    var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));
+                    var cellFromGridStructure = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));

-                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
+                    if (cellFromGridStructure.hasMinimumSize()) {

-                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
-                    if (cell.hasMinimumSize()) {
-                        row.add(cell);
+                        cells.stream()
+                                .map(originalCell -> new CellWithIntersection(originalCell, RectangleTransformations.calculateIntersectedArea(cellFromGridStructure, originalCell)))
+                                .filter(cellWithIntersection -> cellWithIntersection.intersectedArea > 0)
+                                .filter(cellWithIntersection -> cellWithIntersection.originalCell.getArea() > cellWithIntersection.intersectedArea * CELL_AREA_CONTAINED_THRESHOLD)
+                                .max(Comparator.comparing(CellWithIntersection::intersectedArea))
+                                .map(CellWithIntersection::originalCell)
+                                .ifPresent(matchingCell -> cellFromGridStructure.getTextBlocks().addAll(matchingCell.getTextBlocks()));
+
+                        row.add(cellFromGridStructure);
                    }
                }
                prevX = x;
            }

-            if (prevY != null && prevX != null && !row.isEmpty()) {
-                matrix.add(row);
+            // exclude empty rows and rows where all text blocks are empty
+            if (prevY != null && prevX != null && !row.isEmpty() && !row.stream()
+                    .allMatch(cell -> cell.getTextBlocks().isEmpty())) {
+
+                rowsOfCells.add(row);
            }
            prevY = y;
        }

-        Collections.reverse(matrix);
+        Collections.reverse(rowsOfCells);

-        return matrix;
-    }
-
-
-
-    public boolean intersects(Cell cell1, Cell cell2) {
-        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
-            return false;
+        // now cells are removed which are part of a column without any text blocks
+        // this is done by first computing the inverse matrix which contains call columns of cells
+        // then the column indices that have to be removed are determined
+        List<List<Cell>> columnsOfCells = new ArrayList<>();
+        int maxRowLength = rowsOfCells.stream()
+                .map(List::size)
+                .max(java.util.Comparator.naturalOrder())
+                .orElse(0);
+        for (int i = 0; i < maxRowLength; i++) {
+            columnsOfCells.add(new ArrayList<>());
        }
-        double x0 = cell1.getX() + 2;
-        double y0 = cell1.getY() + 2;
-        return (cell2.x + cell2.width > x0 &&
-                cell2.y + cell2.height > y0 &&
-                cell2.x < x0 + cell1.getWidth() -2 &&
-                cell2.y < y0 + cell1.getHeight() -2);
+
+        for (List<Cell> row : rowsOfCells) {
+            for (int j = 0; j < row.size(); j++) {
+                columnsOfCells.get(j).add(row.get(j));
+            }
+        }
+
+        List<Integer> columnIndicesToRemove = new ArrayList<>();
+        int columnIndex = 0;
+        for (List<Cell> col : columnsOfCells) {
+            if (col.stream()
+                    .allMatch(cell -> cell.getTextBlocks().isEmpty())) {
+                columnIndicesToRemove.add(columnIndex);
+            }
+            columnIndex++;
+        }
+        columnIndicesToRemove.sort(Collections.reverseOrder());
+
+        // update all rows so that the values of the empty columns get removed
+        var rowsOfCellsBefore = new ArrayList<>(rowsOfCells);
+        rowsOfCells = new ArrayList<>();
+        for (List<Cell> row : rowsOfCellsBefore) {
+            var updatedRow = new ArrayList<>(row);
+            columnIndicesToRemove.forEach(idxToRemove -> updatedRow.remove(updatedRow.get(idxToRemove)));
+            rowsOfCells.add(updatedRow);
+        }
+
+        return rowsOfCells;
    }


+    private void addCellToRowAndCol(Cell cell, int row, int col) {
+
+        unrotatedRowCount = Math.max(unrotatedRowCount, row + 1);
+        unrotatedColCount = Math.max(unrotatedColCount, col + 1);
+
+        CellPosition cp = new CellPosition(row, col);
+        cellTreeMap.put(cp, cell);
+
+    }
+

    @Override
    public String getText() {
@@ -314,7 +365,7 @@ public class TablePageBlock extends AbstractPageBlock {
                            if (!first) {
                                sb.append("\n");
                            }
-                            sb.append('\"').append(textBlock.getText().replaceAll("\"", "\\\"")).append('\"');
+                            sb.append('\"').append(textBlock.getText().replaceAll("\"", "\"")).append('\"');
                            first = false;
                        }
                    }
@@ -328,8 +379,6 @@ public class TablePageBlock extends AbstractPageBlock {
    }


-
-
    public String getTextAsHtml() {

        StringBuilder sb = new StringBuilder();
@@ -363,4 +412,9 @@ public class TablePageBlock extends AbstractPageBlock {
        return sb.toString();
    }

+
+    record CellWithIntersection(Cell originalCell, double intersectedArea) {
+
+    }
+
 }
@@ -45,9 +45,6 @@ public class RedTextPosition {
    @JsonIgnore
    private String fontName;

-    @JsonIgnore
-    private RedTextPosition parent;
-

    @SneakyThrows
    public static RedTextPosition fromTextPosition(TextPosition textPosition) {
@@ -17,7 +17,6 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.Getter;
 import lombok.NoArgsConstructor;

@EqualsAndHashCode(callSuper = true)
@@ -28,7 +27,6 @@ import lombok.NoArgsConstructor;
 public class TextPageBlock extends AbstractPageBlock {

    @Builder.Default
-    @Getter
    private List<TextPositionSequence> sequences = new ArrayList<>();

    @JsonIgnore
@@ -55,6 +53,9 @@ public class TextPageBlock extends AbstractPageBlock {
    @JsonIgnore
    private PageBlockType classification;

+    @JsonIgnore
+    private boolean toDuplicate;
+

    @JsonIgnore
    public TextDirection getDir() {
@@ -364,7 +365,22 @@ public class TextPageBlock extends AbstractPageBlock {
        }

        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString());
+    }

+
+    public int getNumberOfLines() {
+
+        int numberOfLines = 1;
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : sequences) {
+            if (previous != null) {
+                if (word.getMaxYDirAdj() - previous.getMaxYDirAdj() > word.getTextHeight()) {
+                    numberOfLines++;
+                }
+            }
+            previous = word;
+        }
+        return numberOfLines;
    }


@@ -63,7 +63,6 @@ public class TextPositionSequence implements CharSequence {
        this.rotation = textPositions.get(0).getRotation();
        this.pageHeight = textPositions.get(0).getPageHeight();
        this.pageWidth = textPositions.get(0).getPageWidth();
-        this.isParagraphStart = false;
    }


@@ -0,0 +1,92 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingBox;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class VisualLayoutParsingAdapter {
+
+    private static String SIGNATURES = "signature";
+
+
+    public Map<Integer, List<VisualLayoutParsingResult>> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
+
+        Map<Integer, List<VisualLayoutParsingResult>> tableCells = new HashMap<>();
+        visualLayoutParsingResponse.getData()
+                .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertTableCells(tableData.getBoxes())));
+
+        return tableCells;
+
+    }
+
+
+    public Map<Integer, List<ClassifiedImage>> buildExtractedSignaturesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
+
+        Map<Integer, List<ClassifiedImage>> signatures = new HashMap<>();
+        if (visualLayoutParsingResponse.getData() != null) {
+            visualLayoutParsingResponse.getData()
+                    .forEach(tableData -> signatures.computeIfAbsent(tableData.getPage_idx() + 1, tableCell -> new ArrayList<>())
+                            .addAll(convertSignatures(tableData.getPage_idx()+1, tableData.getBoxes())));
+        }
+        return signatures;
+
+    }
+
+
+    public List<VisualLayoutParsingResult> convertTableCells(List<VisualLayoutParsingBox> tableObjects) {
+
+        List<VisualLayoutParsingResult> parsedTableCells = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            VisualLayoutParsingResult result = new VisualLayoutParsingResult();
+            result.setX0(t.getBox().getX1());
+            result.setX1(t.getBox().getX2());
+            result.setY0(t.getBox().getY1());
+            result.setY1(t.getBox().getY2());
+            result.setWidth(result.getX1() - result.getX0());
+            result.setHeight(result.getY1() - result.getY0());
+            result.setLabel(t.getLabel());
+            parsedTableCells.add(result);
+        });
+
+        return parsedTableCells;
+
+    }
+
+
+    public List<ClassifiedImage> convertSignatures(int pageNumber, List<VisualLayoutParsingBox> tableObjects) {
+
+        List<ClassifiedImage> signatures = new ArrayList<>();
+
+        tableObjects.stream().forEach(t -> {
+            if (t.getLabel().equals(SIGNATURES)) {
+                ClassifiedImage signature = new ClassifiedImage(new Rectangle2D.Float(t.getBox().getX1(),
+                        t.getBox().getY1(),
+                        t.getBox().getX2() - t.getBox().getX1(),
+                        t.getBox().getY2() - t.getBox().getY1()), ImageType.SIGNATURE, true, false, false, pageNumber);
+
+                signatures.add(signature);
+            }
+        });
+
+        return signatures;
+
+    }
+
+}
@@ -0,0 +1,18 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingBox {
+
+    private VisualLayoutParsingBoxValue box;
+    private String label;
+    private float probability;
+
+}
@@ -0,0 +1,19 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingBoxValue {
+
+    private float x1;
+    private float y1;
+    private float x2;
+    private float y2;
+
+}
@@ -0,0 +1,21 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingData {
+
+    private int page_idx;
+
+    private List<VisualLayoutParsingBox> boxes;
+
+}
@@ -0,0 +1,23 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingResponse {
+
+    private String dossierId;
+    private String fileId;
+    private String targetFileExtension;
+    private String responseFileExtension;
+    private String X_TENANT_ID;
+    private List<VisualLayoutParsingData> data;
+
+}
@@ -0,0 +1,22 @@
+package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class VisualLayoutParsingResult {
+
+    private float x0;
+    private float y0;
+    private float x1;
+    private float y1;
+    private float width;
+    private float height;
+    private String label;
+
+}
@@ -25,6 +25,7 @@ public class BodyTextFrameService {
    private static final float RULING_HEIGHT_THRESHOLD = 0.15f; // multiplied with page height. Header/Footer Rulings must be within that border of the page.
    private static final float RULING_WIDTH_THRESHOLD = 0.75f; // multiplied with page width. Header/Footer Rulings must be at least that wide.

+
    public void setBodyTextFrames(ClassificationDocument classificationDocument, LayoutParsingType layoutParsingType) {

        Rectangle bodyTextFrame = calculateBodyTextFrame(classificationDocument.getPages(), classificationDocument.getFontSizeCounter(), false, layoutParsingType);
@@ -132,12 +133,7 @@ public class BodyTextFrameService {
                                               boolean landscape,
                                               LayoutParsingType layoutParsingType) {

-        float approximateHeaderLineCount;
-        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
-            approximateHeaderLineCount = 3.3f;
-        } else {
-            approximateHeaderLineCount = 2.9f;
-        }
+        float approximateHeaderLineCount = 2.9f;

        BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();

@@ -155,8 +151,9 @@ public class BodyTextFrameService {
                        continue;
                    }

-                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                            || MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)) {
+                    if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || MarkedContentUtils.intersects(textBlock,
+                            page.getMarkedContentBboxPerType(),
+                            MarkedContentUtils.FOOTER)) {
                        continue;
                    }

@@ -1,21 +1,21 @@
 package com.knecon.fforesight.service.layoutparser.processor.services;

-import java.awt.geom.Line2D;
+import static com.knecon.fforesight.service.layoutparser.processor.utils.GeometricComparators.X_FIRST_RULING_COMPARATOR;
+
 import java.awt.geom.Point2D;
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
-import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;
+import com.knecon.fforesight.service.layoutparser.processor.utils.UnionFind;

 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -25,25 +25,145 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RulingCleaningService {

-    private static final float THRESHOLD = 6;
+    private static final float THRESHOLD_X_VERTICAL = 1;
+    private static final float THRESHOLD_Y_VERTICAL = 2;
+    private static final float THRESHOLD_X_HORIZONTAL = 2;
+    private static final float THRESHOLD_Y_HORIZONTAL = 3;


    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {

+        Rulings verticalAndHorizontalRulingLines;
+
        if (!rulings.isEmpty()) {
-            snapPoints(rulings);
+            verticalAndHorizontalRulingLines = extractVerticalAndHorizontalRulingLines(rulings);
+        } else {
+            verticalAndHorizontalRulingLines = getRulingsFromParsedCells(tableCells);
        }

+        verticalAndHorizontalRulingLines.verticalLines.sort(X_FIRST_RULING_COMPARATOR);
+        verticalAndHorizontalRulingLines.horizontalLines.sort(X_FIRST_RULING_COMPARATOR);
+        verticalAndHorizontalRulingLines = cleanRulings(verticalAndHorizontalRulingLines);
+
+        return CleanRulings.builder().vertical(verticalAndHorizontalRulingLines.verticalLines()).horizontal(verticalAndHorizontalRulingLines.horizontalLines()).build();
+    }
+
+
+    private Rulings cleanRulings(Rulings rulings) {
+
+        List<List<Rectangle>> groupedOverlappingVerticalRectangles = groupOverlappingRectangles(rulings.verticalLines.stream()
+                                                                                                        .map(RulingCleaningService::getOverlapRectangle)
+                                                                                                        .distinct()
+                                                                                                        .toList());
+        List<Ruling> cleanedVerticalRulings = groupedOverlappingVerticalRectangles.stream()
+                .map(rectList -> getXCenteredRuling(Rectangle.boundingBoxOf(rectList)))
+                .toList();
+
+        List<List<Rectangle>> groupedOverlappingHorizontalRectangles = groupOverlappingRectangles(rulings.horizontalLines.stream()
+                                                                                                          .map(RulingCleaningService::getOverlapRectangle)
+                                                                                                          .distinct()
+                                                                                                          .toList());
+
+        List<Ruling> cleanedHorizontalRulings = groupedOverlappingHorizontalRectangles.stream()
+                .map(rectList -> getYCenteredRuling(Rectangle.boundingBoxOf(rectList)))
+                .collect(Collectors.toList());
+
+        return new Rulings(cleanedVerticalRulings, cleanedHorizontalRulings);
+    }
+
+
+    private List<List<Rectangle>> groupOverlappingRectangles(List<Rectangle> rectangles) {
+
+        UnionFind<Rectangle> unionFind = new UnionFind<>();
+        for (int i = 0; i < rectangles.size(); i++) {
+            for (int j = i + 1; j < rectangles.size(); j++) {
+                Rectangle rectangle1 = rectangles.get(i);
+                Rectangle rectangle2 = rectangles.get(j);
+
+                // we can stop early when we are too far off because of x-y-sorting
+                if(rectangle1.getRight() < rectangle2.getLeft() && rectangle1.getBottom() < rectangle2.getTop()) {
+                    break;
+                }
+
+                if (rectangle1.intersects(rectangle2)) {
+                    unionFind.union(rectangle1, rectangle2);
+                }
+            }
+        }
+
+        Map<Rectangle, List<Rectangle>> groups = new HashMap<>();
+        for (Rectangle rectangle : rectangles) {
+            Rectangle root = unionFind.find(rectangle);
+            groups.computeIfAbsent(root, k -> new ArrayList<>()).add(rectangle);
+        }
+        return new ArrayList<>(groups.values());
+    }
+
+
+    private static Rectangle getOverlapRectangle(Ruling ruling) {
+
+        float top;
+        float left;
+        float w;
+        float h;
+
+        if (ruling.x1 < ruling.x2) {
+            left = ruling.x1;
+            w = ruling.x2 - ruling.x1;
+        } else {
+            left = ruling.x2;
+            w = ruling.x1 - ruling.x2;
+        }
+        if (ruling.y1 < ruling.y2) {
+            top = ruling.y1;
+            h = ruling.y2 - ruling.y1;
+        } else {
+            top = ruling.y2;
+            h = ruling.y1 - ruling.y2;
+        }
+
+        if (ruling.horizontal()) {
+            return new Rectangle(top - THRESHOLD_Y_HORIZONTAL, left - THRESHOLD_X_HORIZONTAL, w + 2 * THRESHOLD_X_HORIZONTAL, h + 2 * THRESHOLD_Y_HORIZONTAL);
+        } else {
+            return new Rectangle(top - THRESHOLD_Y_VERTICAL, left - THRESHOLD_X_VERTICAL, w + 2 * THRESHOLD_X_VERTICAL, h + 2 * THRESHOLD_Y_VERTICAL);
+        }
+    }
+
+
+    public static Ruling getXCenteredRuling(Rectangle rectangle) {
+
+        float x = (float) rectangle.getCenterX();
+        float y1 = rectangle.getTop();
+        float y2 = rectangle.getBottom();
+
+        Point2D point1 = new Point2D.Float(x, y1 + THRESHOLD_Y_VERTICAL);
+        Point2D point2 = new Point2D.Float(x, y2 - THRESHOLD_Y_VERTICAL);
+
+        return new Ruling(point1, point2);
+    }
+
+
+    public static Ruling getYCenteredRuling(Rectangle rectangle) {
+
+        float x1 = rectangle.getLeft();
+        float x2 = rectangle.getRight();
+        float y = (float) rectangle.getCenterY();
+
+        Point2D point1 = new Point2D.Float(x1 + THRESHOLD_X_HORIZONTAL, y);
+        Point2D point2 = new Point2D.Float(x2 - THRESHOLD_X_HORIZONTAL, y);
+
+        return new Ruling(point1, point2);
+    }
+
+
+    private Rulings extractVerticalAndHorizontalRulingLines(List<Ruling> rulings) {
+
        List<Ruling> vrs = new ArrayList<>();
        for (Ruling vr : rulings) {
            if (vr.vertical()) {
                vrs.add(vr);
            }
        }
-        if (vrs.isEmpty()) {
-            vrs.addAll(extractVerticalRulings(tableCells));
-        }
-        List<Ruling> verticalRulingLines = collapseOrientedRulings(vrs);

        List<Ruling> hrs = new ArrayList<>();
        for (Ruling hr : rulings) {
@@ -51,98 +171,26 @@ public class RulingCleaningService {
                hrs.add(hr);
            }
        }
-        if (hrs.isEmpty()) {
-            hrs.addAll(extractHorizontalRulings(tableCells));
-        }
-        List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);
-
-        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
+        return new Rulings(vrs, hrs);
    }


-    public void snapPoints(List<? extends Line2D.Float> rulings) {
+    private Rulings getRulingsFromParsedCells(List<TableCells> tableCells) {

-        // collect points and keep a Line -> p1,p2 map
-        Map<Line2D.Float, Point2D[]> linesToPoints = new HashMap<>();
-        List<Point2D> points = new ArrayList<>();
-        for (Line2D.Float r : rulings) {
-            Point2D p1 = r.getP1();
-            Point2D p2 = r.getP2();
-            linesToPoints.put(r, new Point2D[]{p1, p2});
-            points.add(p1);
-            points.add(p2);
-        }
-
-        // snap by X
-        points.sort(Comparator.comparingDouble(Point2D::getX));
-
-        List<List<Point2D>> groupedPoints = new ArrayList<>();
-        groupedPoints.add(new ArrayList<>(Collections.singletonList(points.get(0))));
-
-        for (Point2D p : points.subList(1, points.size() - 1)) {
-            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
-                groupedPoints.get(groupedPoints.size() - 1).add(p);
-            } else {
-                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
-            }
-        }
-
-        for (List<Point2D> group : groupedPoints) {
-            float avgLoc = 0;
-            for (Point2D p : group) {
-                avgLoc += p.getX();
-            }
-            avgLoc /= group.size();
-            for (Point2D p : group) {
-                p.setLocation(avgLoc, p.getY());
-            }
-        }
-        // ---
-
-        // snap by Y
-        points.sort(Comparator.comparingDouble(Point2D::getY));
-
-        groupedPoints = new ArrayList<>();
-        groupedPoints.add(new ArrayList<>(Collections.singletonList(points.get(0))));
-
-        for (Point2D p : points.subList(1, points.size() - 1)) {
-            List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
-                groupedPoints.get(groupedPoints.size() - 1).add(p);
-            } else {
-                groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
-            }
-        }
-
-        for (List<Point2D> group : groupedPoints) {
-            float avgLoc = 0;
-            for (Point2D p : group) {
-                avgLoc += p.getY();
-            }
-            avgLoc /= group.size();
-            for (Point2D p : group) {
-                p.setLocation(p.getX(), avgLoc);
-            }
-        }
-        // ---
-
-        // finally, modify lines
-        for (Map.Entry<Line2D.Float, Point2D[]> ltp : linesToPoints.entrySet()) {
-            Point2D[] p = ltp.getValue();
-            ltp.getKey().setLine(p[0], p[1]);
-        }
+        List<Ruling> vrs = extractVerticalRulingsFromParsedCells(tableCells);
+        List<Ruling> hrs = extractHorizontalRulingsFromParsedCells(tableCells);
+        return new Rulings(vrs, hrs);
    }


-    private Collection<? extends Ruling> extractVerticalRulings(List<TableCells> cvParsedTableCells) {
+    private List<Ruling> extractVerticalRulingsFromParsedCells(List<TableCells> tableCells) {

        List<Ruling> vrs = new ArrayList<>();

-        if (cvParsedTableCells != null) {
-            for (TableCells cvParsedTableCell : cvParsedTableCells) {
-                Ruling leftLine = createRuling(cvParsedTableCell.getX0(), cvParsedTableCell.getX0(), cvParsedTableCell.getY0(), cvParsedTableCell.getY1());
-                Ruling rightLine = createRuling(cvParsedTableCell.getX1(), cvParsedTableCell.getX1(), cvParsedTableCell.getY0(), cvParsedTableCell.getY1());
+        if (tableCells != null) {
+            for (TableCells tableCell : tableCells) {
+                Ruling leftLine = createRuling(tableCell.getX0(), tableCell.getX0(), tableCell.getY0(), tableCell.getY1());
+                Ruling rightLine = createRuling(tableCell.getX1(), tableCell.getX1(), tableCell.getY0(), tableCell.getY1());
                vrs.add(leftLine);
                vrs.add(rightLine);
            }
@@ -151,19 +199,18 @@ public class RulingCleaningService {
    }


-    private Collection<? extends Ruling> extractHorizontalRulings(List<TableCells> cvParsedTableCells) {
+    private List<Ruling> extractHorizontalRulingsFromParsedCells(List<TableCells> tableCells) {

        List<Ruling> hrs = new ArrayList<>();

-        if (cvParsedTableCells != null) {
-            for (TableCells cvParsedTableCell : cvParsedTableCells) {
-                Ruling topLine = createRuling(cvParsedTableCell.getX0(), cvParsedTableCell.getX1(), cvParsedTableCell.getY1(), cvParsedTableCell.getY1());
-                Ruling baseLine = createRuling(cvParsedTableCell.getX0(), cvParsedTableCell.getX1(), cvParsedTableCell.getY0(), cvParsedTableCell.getY0());
+        if (tableCells != null) {
+            for (TableCells tableCell : tableCells) {
+                Ruling topLine = createRuling(tableCell.getX0(), tableCell.getX1(), tableCell.getY1(), tableCell.getY1());
+                Ruling baseLine = createRuling(tableCell.getX0(), tableCell.getX1(), tableCell.getY0(), tableCell.getY0());
                hrs.add(topLine);
                hrs.add(baseLine);
            }
        }
-
        return hrs;
    }

@@ -189,46 +236,8 @@ public class RulingCleaningService {
    }


-    private List<Ruling> collapseOrientedRulings(List<Ruling> lines) {
+    private record Rulings(List<Ruling> verticalLines, List<Ruling> horizontalLines) {

-        int COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT = 1;
-        return collapseOrientedRulings(lines, COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT);
-    }
-
-
-    private List<Ruling> collapseOrientedRulings(List<Ruling> lines, int expandAmount) {
-
-        ArrayList<Ruling> rv = new ArrayList<>();
-        lines.sort((a, b) -> {
-            final float diff = a.getPosition() - b.getPosition();
-            return Float.compare(diff == 0 ? a.getStart() - b.getStart() : diff, 0f);
-        });
-
-        for (Ruling next_line : lines) {
-            Ruling last = rv.isEmpty() ? null : rv.get(rv.size() - 1);
-            // if current line colinear with next, and are "close enough": expand current line
-            if (last != null && DoubleComparisons.feq(next_line.getPosition(), last.getPosition()) && last.nearlyIntersects(next_line, expandAmount)) {
-                final float lastStart = last.getStart();
-                final float lastEnd = last.getEnd();
-
-                final boolean lastFlipped = lastStart > lastEnd;
-                final boolean nextFlipped = next_line.getStart() > next_line.getEnd();
-
-                boolean differentDirections = nextFlipped != lastFlipped;
-                float nextS = differentDirections ? next_line.getEnd() : next_line.getStart();
-                float nextE = differentDirections ? next_line.getStart() : next_line.getEnd();
-
-                final float newStart = lastFlipped ? Math.max(nextS, lastStart) : Math.min(nextS, lastStart);
-                final float newEnd = lastFlipped ? Math.min(nextE, lastEnd) : Math.max(nextE, lastEnd);
-                last.setStartEnd(newStart, newEnd);
-                assert !last.oblique();
-            } else if (next_line.length() == 0) {
-                continue;
-            } else {
-                rv.add(next_line);
-            }
-        }
-        return rv;
    }

 }
@@ -7,6 +7,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;

+import org.apache.logging.log4j.util.Strings;
 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -110,6 +111,20 @@ public class SectionsBuilderService {
    }


+    public void buildParagraphDebugSections(ClassificationDocument document) {
+
+        List<ClassificationSection> sections = new ArrayList<>();
+        for (var page : document.getPages()) {
+            page.getTextBlocks().forEach(block -> {
+                block.setPage(page.getPageNumber());
+                var section = buildTextBlock(List.of(block), Strings.EMPTY);
+                sections.add(section);
+            });
+        }
+        document.setSections(sections);
+    }
+
+
    public void addImagesToSections(ClassificationDocument document) {

        Map<Integer, List<ClassificationSection>> sectionMap = new HashMap<>();
@@ -240,7 +255,7 @@ public class SectionsBuilderService {
    }


-    public ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {
+    private ClassificationSection buildTextBlock(List<AbstractPageBlock> wordBlockList, String lastHeadline) {

        ClassificationSection section = new ClassificationSection();

@@ -1,79 +1,33 @@
 package com.knecon.fforesight.service.layoutparser.processor.services;

-import java.awt.geom.Point2D;
+import static com.knecon.fforesight.service.layoutparser.processor.utils.GeometricComparators.CELL_SIZE_COMPARATOR;
+import static com.knecon.fforesight.service.layoutparser.processor.utils.GeometricComparators.RECTANGLE_SIZE_COMPARATOR;
+
 import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Rectangle;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangularIntersectionFinder;
+import com.knecon.fforesight.service.layoutparser.processor.utils.SpreadsheetFinder;

@Service
 public class TableExtractionService {

-    private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (arg0, arg1) -> {
-
-        int rv = 0;
-        float arg0X = DoubleComparisons.round(arg0.getX(), 2);
-        float arg0Y = DoubleComparisons.round(arg0.getY(), 2);
-        float arg1X = DoubleComparisons.round(arg1.getX(), 2);
-        float arg1Y = DoubleComparisons.round(arg1.getY(), 2);
-
-        if (arg0X > arg1X) {
-            rv = 1;
-        } else if (arg0X < arg1X) {
-            rv = -1;
-        } else if (arg0Y > arg1Y) {
-            rv = 1;
-        } else if (arg0Y < arg1Y) {
-            rv = -1;
-        }
-        return rv;
-    };
-    private static final Comparator<Point2D> POINT_COMPARATOR = (arg0, arg1) -> {
-
-        int rv = 0;
-        float arg0X = DoubleComparisons.round(arg0.getX(), 2);
-        float arg0Y = DoubleComparisons.round(arg0.getY(), 2);
-        float arg1X = DoubleComparisons.round(arg1.getX(), 2);
-        float arg1Y = DoubleComparisons.round(arg1.getY(), 2);
-
-        if (arg0Y > arg1Y) {
-            rv = 1;
-        } else if (arg0Y < arg1Y) {
-            rv = -1;
-        } else if (arg0X > arg1X) {
-            rv = 1;
-        } else if (arg0X < arg1X) {
-            rv = -1;
-        }
-        return rv;
-    };
-
-
-    public boolean contains(Cell cell, double x, double y, double w, double h) {
-
-        if (cell.isEmpty() || w <= 0 || h <= 0) {
-            return false;
-        }
-        double x0 = cell.getX();
-        double y0 = cell.getY();
-        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
-    }
+    private static final int MAX_TABLE_CONTAINED_CELLS_WITH_TEXT = 1;
+    private static final int TEXT_BLOCK_CONTAINMENT_TOLERANCE = 2;
+    private static final double TABLE_UNIFORMITY_THRESHOLD = 0.7;


    /**
@@ -86,272 +40,104 @@ public class TableExtractionService {
     * <p>
     * DirAdj (Text direction adjusted) values can not be used here.
     *
-     * @param cleanRulings The lines used to build the table.
-     * @param page         Page object that contains textblocks and statistics.
+     * @param emptyCells The cells used to build the table.
+     * @param page       Page object that contains textblocks and statistics.
     */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {

-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
+    public void extractTables(List<Cell> emptyCells, ClassificationPage page) {

-        List<TextPageBlock> toBeRemoved = new ArrayList<>();
+        // sort cells by size (height * width) ascending so that textBlocks are always assigned to the smallest cells that contain them
+        emptyCells.sort(CELL_SIZE_COMPARATOR);

        for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
            TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
-            for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && contains(cell,
-                        textBlock.getPdfMinX(),
-                        textBlock.getPdfMinY(),
-                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
-                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
+            for (Cell cell : emptyCells) {
+                if (cell.hasMinimumSize() && doesCellContainTextBlock(cell, textBlock)) {
                    cell.addTextBlock(textBlock);
-                    toBeRemoved.add(textBlock);
                    break;
                }
            }
        }

-        cells = new ArrayList<>(new HashSet<>(cells));
+        var cells = new ArrayList<>(new HashSet<>(emptyCells));
        DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);

-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);
+        List<Rectangle> spreadsheetAreas = SpreadsheetFinder.findSpreadsheetsFromCells(cells);
+        // sort spreadsheetAreas by size (height * width) ascending so that cells are placed in the smallest tables first
+        // this way no cell duplication occurs when tables are contained in other tables and only the most inner table contains the cells
+        spreadsheetAreas.sort(RECTANGLE_SIZE_COMPARATOR);

        List<TablePageBlock> tables = new ArrayList<>();
        for (Rectangle area : spreadsheetAreas) {

-            List<Cell> overlappingCells = new ArrayList<>();
+            List<Cell> containedCells = new ArrayList<>();
            for (Cell c : cells) {
-                if (c.hasMinimumSize() && c.intersects(area)) {
-                    overlappingCells.add(c);
+                if (c.hasMinimumSize() && area.contains(c)) {
+                    containedCells.add(c);
                }
            }
-            tables.add(new TablePageBlock(overlappingCells, area, page.getRotation()));
+
+            var containedCellsWithText = containedCells.stream().filter(cell -> !cell.getTextBlocks().isEmpty()).toList();
+
+            // verify if table would contain fewer cells with text than the threshold allows
+            if (containedCellsWithText.size() >= MAX_TABLE_CONTAINED_CELLS_WITH_TEXT && checkIfTableCellsAreUniform(containedCells)) {
+                tables.add(new TablePageBlock(containedCells, area, page.getRotation()));
+                cells.removeAll(containedCells);
+            }
        }

        for (TablePageBlock table : tables) {
            int position = -1;

-            Iterator<AbstractPageBlock> itty = page.getTextBlocks().iterator();
-            while (itty.hasNext()) {
-                AbstractPageBlock textBlock = itty.next();
-                if (textBlock instanceof TextPageBlock ? table.containsBlock((TextPageBlock) textBlock) : table.contains(textBlock) && position == -1) {
-                    position = page.getTextBlocks().indexOf(textBlock);
+            for (AbstractPageBlock pageBlock : page.getTextBlocks()) {
+                if (pageBlock instanceof TextPageBlock ? table.containsBlock((TextPageBlock) pageBlock) : table.contains(pageBlock) && position == -1) {
+                    position = page.getTextBlocks().indexOf(pageBlock);
                }
            }
            if (position != -1) {
                page.getTextBlocks().add(position, table);
+
+                var toBeRemoved = table.getCells().stream().map(Cell::getTextBlocks).flatMap(List::stream).toList();
+                // remove text blocks from the page that were also added with the table (from its contained cells)
+                page.getTextBlocks().removeAll(toBeRemoved);
            }
        }
-
-        page.getTextBlocks().removeAll(toBeRemoved);
    }


-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    private boolean checkIfTableCellsAreUniform(List<Cell> containedCells) {

-        // Fix for 211.pdf
-        for (Ruling r : horizontalRulingLines) {
-            if (r.getX2() < r.getX1()) {
-                double a = r.getX2();
-                r.x2 = (float) r.getX1();
-                r.x1 = (float) a;
-            }
+        if (containedCells.size() <= 2) {
+            return true;
        }

-        List<Cell> cellsFound = new ArrayList<>();
-        Map<Point2D, Ruling[]> intersectionPoints = Ruling.findIntersections(horizontalRulingLines, verticalRulingLines);
-        List<Point2D> intersectionPointsList = new ArrayList<>(intersectionPoints.keySet());
-        intersectionPointsList.sort(POINT_COMPARATOR);
+        Map<Long, List<Long>> cellsGroupedByRoundedWidth = containedCells.stream()
+                .map(Rectangle::getWidth)
+                .map(size -> Math.round(size / 10.0) * 10)
+                .collect(Collectors.groupingBy(Long::longValue));

-        for (int i = 0; i < intersectionPointsList.size(); i++) {
-            Point2D topLeft = intersectionPointsList.get(i);
-            Ruling[] hv = intersectionPoints.get(topLeft);
-
-            // CrossingPointsDirectlyBelow( topLeft );
-            List<Point2D> xPoints = new ArrayList<>();
-            // CrossingPointsDirectlyToTheRight( topLeft );
-            List<Point2D> yPoints = new ArrayList<>();
-
-            for (Point2D p : intersectionPointsList.subList(i, intersectionPointsList.size())) {
-                if (p.getX() == topLeft.getX() && p.getY() > topLeft.getY()) {
-                    xPoints.add(p);
-                }
-                if (p.getY() == topLeft.getY() && p.getX() > topLeft.getX()) {
-                    yPoints.add(p);
-                }
-            }
-            outer:
-            for (Point2D xPoint : xPoints) {
-                // is there a vertical edge b/w topLeft and xPoint?
-                if (!hv[1].equals(intersectionPoints.get(xPoint)[1])) {
-                    continue;
-                }
-                for (Point2D yPoint : yPoints) {
-                    // is there an horizontal edge b/w topLeft and yPoint ?
-                    if (!hv[0].equals(intersectionPoints.get(yPoint)[0])) {
-                        continue;
-                    }
-                    Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY());
-                    if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(
-                            intersectionPoints.get(yPoint)[1])) {
-                        cellsFound.add(new Cell(topLeft, btmRight));
-                        break outer;
-                    }
-                }
-            }
-        }
-
-        // TODO create cells for vertical ruling lines with aligned endpoints at the top/bottom of a grid
-        // that aren't connected with an horizontal ruler?
-        // see: https://github.com/jazzido/tabula-extractor/issues/78#issuecomment-41481207
-
-        return cellsFound;
+        return (double) cellsGroupedByRoundedWidth.size() / containedCells.size() <= TABLE_UNIFORMITY_THRESHOLD;
    }


-    private List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
-        // via: http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon
-        List<Rectangle> rectangles = new ArrayList<>();
-        Set<Point2D> pointSet = new HashSet<>();
-        Map<Point2D, Point2D> edgesH = new HashMap<>();
-        Map<Point2D, Point2D> edgesV = new HashMap<>();
-        int i = 0;
+    private boolean doesCellContainTextBlock(Cell cell, TextPageBlock textBlock) {

-        for (Rectangle cell : cells) {
-            for (Point2D pt : cell.getPoints()) {
-                if (pointSet.contains(pt)) { // shared vertex, remove it
-                    pointSet.remove(pt);
-                } else {
-                    pointSet.add(pt);
-                }
-            }
+        double x = textBlock.getPdfMinX();
+        double y = textBlock.getPdfMinY();
+        double w = textBlock.getPdfMaxX() - textBlock.getPdfMinX();
+        double h = textBlock.getPdfMaxY() - textBlock.getPdfMinY();
+        if (cell.isEmpty() || w <= 0 || h <= 0) {
+            return false;
        }
-
-        // X first sort
-        List<Point2D> pointsSortX = new ArrayList<>(pointSet);
-        pointsSortX.sort(X_FIRST_POINT_COMPARATOR);
-        // Y first sort
-        List<Point2D> pointsSortY = new ArrayList<>(pointSet);
-        pointsSortY.sort(POINT_COMPARATOR);
-
-        while (i < pointSet.size()) {
-            float currY = (float) pointsSortY.get(i).getY();
-            while (i < pointSet.size() && DoubleComparisons.feq(pointsSortY.get(i).getY(), currY)) {
-                edgesH.put(pointsSortY.get(i), pointsSortY.get(i + 1));
-                edgesH.put(pointsSortY.get(i + 1), pointsSortY.get(i));
-                i += 2;
-            }
-        }
-
-        i = 0;
-        while (i < pointSet.size()) {
-            float currX = (float) pointsSortX.get(i).getX();
-            while (i < pointSet.size() && DoubleComparisons.feq(pointsSortX.get(i).getX(), currX)) {
-                edgesV.put(pointsSortX.get(i), pointsSortX.get(i + 1));
-                edgesV.put(pointsSortX.get(i + 1), pointsSortX.get(i));
-                i += 2;
-            }
-        }
-
-        // Get all the polygons
-        List<List<PolygonVertex>> polygons = new ArrayList<>();
-        Point2D nextVertex;
-        while (!edgesH.isEmpty()) {
-            ArrayList<PolygonVertex> polygon = new ArrayList<>();
-            Point2D first = edgesH.keySet().iterator().next();
-            polygon.add(new PolygonVertex(first, Direction.HORIZONTAL));
-            edgesH.remove(first);
-
-            while (true) {
-                PolygonVertex curr = polygon.get(polygon.size() - 1);
-                PolygonVertex lastAddedVertex;
-                if (curr.direction == Direction.HORIZONTAL) {
-                    nextVertex = edgesV.get(curr.point);
-                    edgesV.remove(curr.point);
-                    lastAddedVertex = new PolygonVertex(nextVertex, Direction.VERTICAL);
-                } else {
-                    nextVertex = edgesH.get(curr.point);
-                    edgesH.remove(curr.point);
-                    lastAddedVertex = new PolygonVertex(nextVertex, Direction.HORIZONTAL);
-                }
-                polygon.add(lastAddedVertex);
-
-                if (lastAddedVertex.equals(polygon.get(0))) {
-                    // closed polygon
-                    polygon.remove(polygon.size() - 1);
-                    break;
-                }
-            }
-
-            for (PolygonVertex vertex : polygon) {
-                edgesH.remove(vertex.point);
-                edgesV.remove(vertex.point);
-            }
-            polygons.add(polygon);
-        }
-
-        // calculate grid-aligned minimum area rectangles for each found polygon
-        for (List<PolygonVertex> poly : polygons) {
-            float top = Float.MAX_VALUE;
-            float left = Float.MAX_VALUE;
-            float bottom = Float.MIN_VALUE;
-            float right = Float.MIN_VALUE;
-            for (PolygonVertex pt : poly) {
-                top = (float) Math.min(top, pt.point.getY());
-                left = (float) Math.min(left, pt.point.getX());
-                bottom = (float) Math.max(bottom, pt.point.getY());
-                right = (float) Math.max(right, pt.point.getX());
-            }
-            rectangles.add(new Rectangle(top, left, right - left, bottom - top));
-        }
-
-        return rectangles;
+        double x0 = cell.getX();
+        double y0 = cell.getY();
+        return (x >= x0 - TEXT_BLOCK_CONTAINMENT_TOLERANCE && y >= y0 - TEXT_BLOCK_CONTAINMENT_TOLERANCE && (x + w) <= x0 + cell.getWidth() + 2 * TEXT_BLOCK_CONTAINMENT_TOLERANCE && (y + h) <= y0 + cell.getHeight() + 2 * TEXT_BLOCK_CONTAINMENT_TOLERANCE);
    }


-    private enum Direction {
-        HORIZONTAL,
-        VERTICAL
-    }
-
-    static class PolygonVertex {
-
-        Point2D point;
-        Direction direction;
-
-
-        PolygonVertex(Point2D point, Direction direction) {
-
-            this.direction = direction;
-            this.point = point;
-        }
-
-
-        @Override
-        public boolean equals(Object other) {
-
-            if (this == other) {
-                return true;
-            }
-            if (!(other instanceof PolygonVertex)) {
-                return false;
-            }
-            return this.point.equals(((PolygonVertex) other).point);
-        }
-
-
-        @Override
-        public int hashCode() {
-
-            return this.point.hashCode();
-        }
-
-
-        @Override
-        public String toString() {
-
-            return String.format("%s[point=%s,direction=%s]", this.getClass().getName(), this.point.toString(), this.direction.toString());
-        }
+    public static List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {

+        return RectangularIntersectionFinder.find(horizontalRulingLines, verticalRulingLines).stream().map(Cell::new).collect(Collectors.toList());
    }

 }
@@ -0,0 +1,408 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.blockification;
+
+import static java.util.stream.Collectors.toSet;
+
+import java.awt.geom.Point2D;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Set;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.FloatFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmentationService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.QuickSort;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
+import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionSequenceComparator;
+
+import lombok.RequiredArgsConstructor;
+
+@SuppressWarnings("all")
+@Service
+@RequiredArgsConstructor
+public class DocstrumBlockificationService {
+
+    private final DocstrumSegmentationService docstrumSegmentationService;
+
+    static final float THRESHOLD = 1f;
+
+
+    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Cell> cells, boolean xyOrder) {
+
+        // Underlined or strikethrough are also in rulings but we dont want to split blocks with them so we use cells.
+        List<Ruling> usedHorizonalRulings = new ArrayList<>();
+        List<Ruling> usedVerticalRulings = new ArrayList<>();
+
+        cells.forEach(cell -> {
+            usedHorizonalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y), new Point2D.Float(cell.x + cell.width, cell.y)));
+            usedHorizonalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y + cell.height), new Point2D.Float(cell.x + cell.width, cell.y + cell.height)));
+            usedVerticalRulings.add(new Ruling(new Point2D.Float(cell.x, cell.y), new Point2D.Float(cell.x, cell.y + cell.height)));
+            usedVerticalRulings.add(new Ruling(new Point2D.Float(cell.x + cell.width, cell.y), new Point2D.Float(cell.x + cell.width, cell.y + cell.height)));
+        });
+
+        List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
+        var zones = docstrumSegmentationService.segmentPage(textPositions, xyOrder);
+        zones.forEach(zone -> {
+
+            List<TextPositionSequence> textPositionSequences = new ArrayList<>();
+            zone.getLines().forEach(line -> {
+                line.getWords().forEach(word -> {
+                    textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), word.getPage()));
+                });
+            });
+
+            abstractPageBlocks.addAll(splitZonesAtRulings(textPositionSequences, usedHorizonalRulings, usedVerticalRulings));
+        });
+
+        return new ClassificationPage(abstractPageBlocks);
+    }
+
+
+    public void combineBlocks(ClassificationPage page) {
+
+        mergeZones(page.getTextBlocks());
+
+        TextPageBlock previous = new TextPageBlock();
+        ListIterator<AbstractPageBlock> itty = page.getTextBlocks().listIterator();
+        while (itty.hasNext()) {
+            AbstractPageBlock block = itty.next();
+            if (block instanceof TablePageBlock) {
+                continue;
+            }
+            TextPageBlock current = (TextPageBlock) block;
+
+            if (previous != null && !previous.getSequences().isEmpty()) {
+
+                if (current.getDir() == previous.getDir() //
+                        && previous.getNumberOfLines() >= 2 && current.getNumberOfLines() >= 2 //
+                        && previous.intersectsY(current) //
+                        && !hasBetween(current, previous, page.getTextBlocks()) //
+                        && numberOfYIntersections(current, previous, page.getTextBlocks()) == 0) {
+
+                    previous.getSequences().addAll(current.getSequences());
+                    previous = buildTextBlock(previous.getSequences(), 0);
+                    previous.setToDuplicate(true);
+                    itty.remove();
+                    itty.previous();
+                    itty.set(previous);
+                    itty.next();
+                    continue;
+                }
+
+                if (current.getDir() == previous.getDir() && (previous.almostIntersects(current, 0, 0))) {
+
+                    previous.getSequences().addAll(current.getSequences());
+                    boolean toDuplicate = previous.isToDuplicate();
+                    previous = buildTextBlock(previous.getSequences(), 0);
+                    previous.setToDuplicate(toDuplicate);
+                    itty.remove();
+                    itty.previous();
+                    itty.set(previous);
+                    itty.next();
+                    continue;
+                }
+
+                if (current.getDir() == previous.getDir() //
+                        && (Math.abs(previous.getMaxY() - current.getMaxY()) < THRESHOLD || Math.abs(previous.getMinY() - current.getMinY()) < THRESHOLD) //
+                        && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
+                        && !hasBetween(current, previous, page.getTextBlocks()) && numberOfYIntersections(current, previous, page.getTextBlocks()) <= 4) {
+
+                    previous.getSequences().addAll(current.getSequences());
+                    previous = buildTextBlock(previous.getSequences(), 0);
+                    itty.remove();
+                    itty.previous();
+                    itty.set(previous);
+                    itty.next();
+                    continue;
+                }
+
+                if (current.getDir() == previous.getDir() //
+                        && current.intersectsY(previous) //
+                        && (previous.getNumberOfLines() == 1 && current.getNumberOfLines() >= 1 || previous.getNumberOfLines() == 2 && current.getNumberOfLines() == 1) //
+                        && !hasBetween(current, previous, page.getTextBlocks()) //
+                        && numberOfYIntersections(current, previous, page.getTextBlocks()) <= 0) {
+                    previous.getSequences().addAll(current.getSequences());
+                    previous = buildTextBlock(previous.getSequences(), 0);
+                    itty.remove();
+                    itty.previous();
+                    itty.set(previous);
+                    itty.next();
+                    continue;
+                }
+
+            }
+            previous = current;
+        }
+
+        mergeZones(page.getTextBlocks());
+
+    }
+
+
+    private boolean hasBetween(TextPageBlock block, TextPageBlock other, List<AbstractPageBlock> allBlocks) {
+
+        for (AbstractPageBlock current : allBlocks) {
+
+            if (current == other || current == block) {
+                continue;
+            }
+
+            if (other.intersectsY(current) && other.getMaxX() <= current.getMinX() && current.getMaxX() <= block.getMinX()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+
+    private int numberOfYIntersections(TextPageBlock block, TextPageBlock other, List<AbstractPageBlock> allBlocks) {
+
+        double minY = Math.min(block.getMinY(), other.getMinY());
+        double maxY = Math.min(block.getMaxY(), other.getMaxY());
+
+        int numberOfYIntersections = 0;
+        for (AbstractPageBlock current : allBlocks) {
+
+            if (current == other || current == block) {
+                continue;
+            }
+
+            if (minY <= current.getMaxY() && maxY >= current.getMinY()) {
+                numberOfYIntersections++;
+            }
+        }
+
+        return numberOfYIntersections;
+    }
+
+
+    public void mergeZones(List<AbstractPageBlock> zones) {
+
+        ListIterator<AbstractPageBlock> itty = zones.listIterator();
+        Set<AbstractPageBlock> toRemove = new HashSet<>();
+        while (itty.hasNext()) {
+            AbstractPageBlock block = itty.next();
+            if (block instanceof TablePageBlock) {
+                continue;
+            }
+
+            TextPageBlock current = (TextPageBlock) block;
+
+            if (current.isToDuplicate()) {
+                continue;
+            }
+
+            for (int i = 0; i < zones.size(); i++) {
+
+                if (toRemove.contains(zones.get(i))) {
+                    continue;
+                }
+                if (zones.get(i) == current) {
+                    continue;
+                }
+                if (zones.get(i) instanceof TablePageBlock) {
+                    continue;
+                }
+
+                TextPageBlock inner = (TextPageBlock) zones.get(i);
+
+                if (inner.isToDuplicate()) {
+                    continue;
+                }
+
+                if (current.getDir() == inner.getDir() && current.almostIntersects(inner, 0, 0)) {
+
+                    current.getSequences().addAll(inner.getSequences());
+                    QuickSort.sort(current.getSequences(), new TextPositionSequenceComparator());
+                    current = buildTextBlock(current.getSequences(), 0);
+                    toRemove.add(inner);
+                    itty.set(current);
+                }
+            }
+        }
+        zones.removeAll(toRemove);
+    }
+
+
+    public List<AbstractPageBlock> splitZonesAtRulings(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+
+        int indexOnPage = 0;
+        List<TextPositionSequence> chunkWords = new ArrayList<>();
+        List<AbstractPageBlock> chunkBlockList = new ArrayList<>();
+
+        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
+        TextPositionSequence prev = null;
+
+        for (TextPositionSequence word : textPositions) {
+
+            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
+            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
+
+            if (prev != null && (splitByDir || isSplitByRuling)) {
+
+                TextPageBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+                indexOnPage++;
+
+                chunkBlockList.add(cb1);
+                chunkWords = new ArrayList<>();
+
+                minX = 1000;
+                maxX = 0;
+                minY = 1000;
+                maxY = 0;
+                prev = null;
+            }
+
+            chunkWords.add(word);
+
+            prev = word;
+            if (word.getMinXDirAdj() < minX) {
+                minX = word.getMinXDirAdj();
+            }
+            if (word.getMaxXDirAdj() > maxX) {
+                maxX = word.getMaxXDirAdj();
+            }
+            if (word.getMinYDirAdj() < minY) {
+                minY = word.getMinYDirAdj();
+            }
+            if (word.getMaxYDirAdj() > maxY) {
+                maxY = word.getMaxYDirAdj();
+            }
+        }
+
+        TextPageBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+        if (cb1 != null) {
+            chunkBlockList.add(cb1);
+        }
+
+        return chunkBlockList;
+    }
+
+
+    private boolean equalsWithThreshold(float f1, float f2) {
+
+        return Math.abs(f1 - f2) < THRESHOLD;
+    }
+
+
+    private TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+
+        TextPageBlock textBlock = null;
+
+        FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
+        FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
+        FloatFrequencyCounter spaceFrequencyCounter = new FloatFrequencyCounter();
+        StringFrequencyCounter fontFrequencyCounter = new StringFrequencyCounter();
+        StringFrequencyCounter styleFrequencyCounter = new StringFrequencyCounter();
+
+        for (TextPositionSequence wordBlock : wordBlockList) {
+
+            lineHeightFrequencyCounter.add(wordBlock.getTextHeight());
+            fontSizeFrequencyCounter.add(wordBlock.getFontSize());
+            spaceFrequencyCounter.add(wordBlock.getSpaceWidth());
+            fontFrequencyCounter.add(wordBlock.getFont());
+            styleFrequencyCounter.add(wordBlock.getFontStyle());
+
+            if (textBlock == null) {
+                textBlock = new TextPageBlock(wordBlock.getMinXDirAdj(),
+                        wordBlock.getMaxXDirAdj(),
+                        wordBlock.getMinYDirAdj(),
+                        wordBlock.getMaxYDirAdj(),
+                        wordBlockList,
+                        wordBlock.getRotation());
+            } else {
+                TextPageBlock spatialEntity = textBlock.union(wordBlock);
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
+            }
+        }
+
+        if (textBlock != null) {
+            textBlock.setMostPopularWordFont(fontFrequencyCounter.getMostPopular());
+            textBlock.setMostPopularWordStyle(styleFrequencyCounter.getMostPopular());
+            textBlock.setMostPopularWordFontSize(fontSizeFrequencyCounter.getMostPopular());
+            textBlock.setMostPopularWordHeight(lineHeightFrequencyCounter.getMostPopular());
+            textBlock.setMostPopularWordSpaceWidth(spaceFrequencyCounter.getMostPopular());
+            textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest());
+        }
+
+        if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getMinYDirAdj(), 3)).collect(toSet()).size() == 1) {
+            textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getMinXDirAdj));
+        }
+        return textBlock;
+    }
+
+
+    private boolean isSplitByRuling(float minX,
+                                    float minY,
+                                    float maxX,
+                                    float maxY,
+                                    TextPositionSequence word,
+                                    List<Ruling> horizontalRulingLines,
+                                    List<Ruling> verticalRulingLines) {
+
+        return isSplitByRuling(maxX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMinYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(maxX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMinYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight());
+    }
+
+
+    private boolean isSplitByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines, float dir, float pageWidth, float pageHeight) {
+
+        for (Ruling ruling : rulingLines) {
+            var line = RulingTextDirAdjustUtil.convertToDirAdj(ruling, dir, pageWidth, pageHeight);
+            if (line.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+
+    private double round(float value, int decimalPoints) {
+
+        var d = Math.pow(10, decimalPoints);
+        return Math.round(value * d) / d;
+    }
+
+}
@@ -57,7 +57,7 @@ public class RedactManagerBlockificationService {
            boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());

-            if (prev != null && (splitByDir || isSplitByRuling)) {
+            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {

                Orientation prevOrientation = null;
                if (!chunkBlockList.isEmpty()) {
@@ -167,7 +167,7 @@ public class RedactManagerBlockificationService {
    }


-    public TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    private TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {

        TextPageBlock textBlock = null;

@@ -1,330 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.blockification;
-
-
-// TODO: figure out, why this fails the build
-// import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.Orientation;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;
-
-@Service
-@SuppressWarnings("all")
-public class TaasBlockificationService {
-
-    private static final float THRESHOLD = 1f;
-    private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f; // multiplied with text height
-    private static final float INTERSECTS_Y_THRESHOLD = 4;// 2 * HEIGHT_PADDING // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
-    private static final int X_GAP_SPLIT_CONSTANT = 50;
-    public static final int X_ALIGNMENT_THRESHOLD = 1;
-    public static final int NEGATIVE_X_GAP_THRESHOLD = -5;
-
-    private Pattern listIdentifier = Pattern.compile("^(?:(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)]))|\\uF0B7", Pattern.CASE_INSENSITIVE);
-
-
-    /**
-     * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
-     * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
-     * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
-     *
-     * @param textPositions         The words of a page.
-     * @param horizontalRulingLines Horizontal table lines.
-     * @param verticalRulingLines   Vertical table lines.
-     * @return ClassificationPage object that contains the Textblock and text statistics.
-     */
-    public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
-
-        List<TextPageBlock> classificationTextBlocks = constructFineGranularTextPageBlocks(textPositions, horizontalRulingLines, verticalRulingLines);
-        classificationTextBlocks = mergeTextPageBlocksAligningX(classificationTextBlocks);
-        classificationTextBlocks = mergeIntersectingTextBlocksUntilConvergence(classificationTextBlocks);
-
-        return new ClassificationPage(new ArrayList<>(classificationTextBlocks.stream().map(classificationTextBlock -> (AbstractPageBlock) classificationTextBlock).toList()));
-    }
-
-
-    private List<TextPageBlock> mergeIntersectingTextBlocksUntilConvergence(List<TextPageBlock> classificationTextBlocks) {
-
-        int currentSize = classificationTextBlocks.size();
-        while (true) {
-            classificationTextBlocks = mergeTextPageBlocksAlmostIntersecting(classificationTextBlocks);
-            if (classificationTextBlocks.size() == currentSize) {
-                break;
-            }
-            currentSize = classificationTextBlocks.size();
-        }
-        return classificationTextBlocks;
-    }
-
-
-    private List<TextPageBlock> mergeTextPageBlocksAligningX(List<TextPageBlock> classificationTextBlocks) {
-
-        if (classificationTextBlocks.isEmpty()) {
-            return new ArrayList<>();
-        }
-        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
-        List<TextPageBlock> currentTextBlocksToMerge = new LinkedList<>();
-        textBlocksToMerge.add(currentTextBlocksToMerge);
-        TextPageBlock previousTextBlock = null;
-        Float lastLineGap = null;
-        for (TextPageBlock currentTextBlock : classificationTextBlocks) {
-            if (previousTextBlock == null) {
-                currentTextBlocksToMerge.add(currentTextBlock);
-                previousTextBlock = currentTextBlock;
-                continue;
-            }
-
-
-            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
-            boolean isListIdentifier = listIdentifierPattern.find();
-
-            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-
-            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
-//            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-
-            boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
-            boolean alignsXLeft = Math.abs(currentTextBlock.getPdfMinX() - previousTextBlock.getPdfMinX()) < X_ALIGNMENT_THRESHOLD;
-//            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < yGap;
-            if (yGap && sameFont && !isListIdentifier) {
-                currentTextBlocksToMerge.add(currentTextBlock);
-
-            } else {
-                currentTextBlocksToMerge = new LinkedList<>();
-                currentTextBlocksToMerge.add(currentTextBlock);
-                textBlocksToMerge.add(currentTextBlocksToMerge);
-            }
-            previousTextBlock = currentTextBlock;
-        }
-        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
-    }
-
-
-    private List<TextPageBlock> mergeTextPageBlocksAlmostIntersecting(List<TextPageBlock> textPageBlocks) {
-
-        Set<TextPageBlock> alreadyMerged = new HashSet<>();
-        List<List<TextPageBlock>> textBlocksToMerge = new LinkedList<>();
-        for (TextPageBlock textPageBlock : textPageBlocks) {
-            if (alreadyMerged.contains(textPageBlock)) {
-                continue;
-            }
-            alreadyMerged.add(textPageBlock);
-            textBlocksToMerge.add(Stream.concat(Stream.of(textPageBlock),
-                            textPageBlocks.stream().filter(textPageBlock2 -> textPageBlock.almostIntersects(textPageBlock2, INTERSECTS_Y_THRESHOLD, 0) && !alreadyMerged.contains(textPageBlock2)).peek(alreadyMerged::add))
-                    .toList());
-        }
-        return textBlocksToMerge.stream().map(TextPageBlock::merge).toList();
-    }
-
-
-    private void assignOrientations(List<TextPageBlock> classificationTextBlocks) {
-
-        Iterator<TextPageBlock> itty = classificationTextBlocks.iterator();
-
-        TextPageBlock previousLeft = null;
-        TextPageBlock previousRight = null;
-        while (itty.hasNext()) {
-            TextPageBlock block = (TextPageBlock) itty.next();
-
-            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
-                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
-                    previousLeft.add(block);
-                    itty.remove();
-                    continue;
-                }
-            }
-
-            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
-                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
-                    previousRight.add(block);
-                    itty.remove();
-                    continue;
-                }
-            }
-
-            if (block.getOrientation().equals(Orientation.LEFT)) {
-                previousLeft = block;
-            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
-                previousRight = block;
-            }
-        }
-
-        itty = classificationTextBlocks.iterator();
-        TextPageBlock previous = null;
-        while (itty.hasNext()) {
-            TextPageBlock block = (TextPageBlock) itty.next();
-
-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(
-                    block.getMaxY(),
-                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
-                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
-                previous.add(block);
-                itty.remove();
-                continue;
-            }
-
-            previous = block;
-        }
-    }
-
-
-    private List<TextPageBlock> constructFineGranularTextPageBlocks(List<TextPositionSequence> textPositions,
-                                                                    List<Ruling> horizontalRulingLines,
-                                                                    List<Ruling> verticalRulingLines) {
-
-        int indexOnPage = 0;
-        List<TextPositionSequence> wordClusterToCombine = new ArrayList<>();
-        List<TextPageBlock> classificationTextBlocks = new ArrayList<>();
-
-        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
-        TextPositionSequence prev = null;
-        // TODO: make static final constant
-
-
-        boolean wasSplitted = false;
-        Float splitX1 = null;
-        for (TextPositionSequence word : textPositions) {
-
-            Matcher listIdentifierPattern = listIdentifier.matcher(word.toString());
-
-            boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-            boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());
-            boolean positiveXGapInline = prev != null && maxX + X_GAP_SPLIT_CONSTANT < word.getMinXDirAdj() && sameLine;
-            boolean negativeXGap = prev != null && word.getMinXDirAdj() - minX < NEGATIVE_X_GAP_THRESHOLD;
-            boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
-            boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
-            boolean splitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
-            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean fontChange = prev != null && (!word.getFont().equals(prev.getFont()) || !word.getFontStyle()
-                    .equals(prev.getFontStyle()) || word.getFontSize() != prev.getFontSize());
-            boolean newline = prev != null && Math.abs(word.getMinYDirAdj() - prev.getMinYDirAdj()) > word.getHeight();
-            boolean isListIdentifier = listIdentifierPattern.matches();
-
-            if (prev != null && (prev.isParagraphStart() || negativeXGap || positiveXGapInline || yGap || startFromTop || splitByRuling || (newline && (fontChange || isListIdentifier)))) {
-//            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
-
-                Orientation prevOrientation = null;
-                if (!classificationTextBlocks.isEmpty()) {
-                    prevOrientation = classificationTextBlocks.get(classificationTextBlocks.size() - X_ALIGNMENT_THRESHOLD).getOrientation();
-                }
-
-                TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);
-
-                classificationTextBlocks.add(classificationTextBlock);
-                wordClusterToCombine = new ArrayList<>();
-
-                if (positiveXGapInline && !splitByRuling) {
-                    wasSplitted = true;
-                    classificationTextBlock.setOrientation(Orientation.LEFT);
-                    splitX1 = word.getMinXDirAdj();
-                } else if (newLineAfterSplit && !splitByRuling) {
-                    wasSplitted = false;
-                    classificationTextBlock.setOrientation(Orientation.RIGHT);
-                    splitX1 = null;
-                } else if (prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (yGap || !startFromTop || !positiveXGapInline || !newLineAfterSplit || !splitByRuling)) {
-                    classificationTextBlock.setOrientation(Orientation.LEFT);
-                }
-
-                minX = 1000;
-                maxX = 0;
-                minY = 1000;
-                maxY = 0;
-                prev = null;
-            }
-
-            wordClusterToCombine.add(word);
-
-            prev = word;
-            if (word.getMinXDirAdj() < minX) {
-                minX = word.getMinXDirAdj();
-            }
-            if (word.getMaxXDirAdj() > maxX) {
-                maxX = word.getMaxXDirAdj();
-            }
-            if (word.getMinYDirAdj() < minY) {
-                minY = word.getMinYDirAdj();
-            }
-            if (word.getMaxYDirAdj() > maxY) {
-                maxY = word.getMaxYDirAdj();
-            }
-        }
-
-        TextPageBlock classificationTextBlock = TextPageBlock.fromTextPositionSequences(wordClusterToCombine);
-        if (classificationTextBlock != null) {
-            classificationTextBlocks.add(classificationTextBlock);
-        }
-        return classificationTextBlocks;
-    }
-
-
-    private boolean equalsWithThreshold(float f1, float f2) {
-
-        return Math.abs(f1 - f2) < THRESHOLD;
-    }
-
-
-    private boolean isSplitByRuling(float minX,
-                                    float minY,
-                                    float maxX,
-                                    float maxY,
-                                    TextPositionSequence word,
-                                    List<Ruling> horizontalRulingLines,
-                                    List<Ruling> verticalRulingLines) {
-
-        return isSplitByRuling(maxX,
-                minY,
-                word.getMinXDirAdj(),
-                word.getMinYDirAdj(),
-                verticalRulingLines,
-                word.getDir().getDegrees(),
-                word.getPageWidth(),
-                word.getPageHeight()) //
-                || isSplitByRuling(minX,
-                minY,
-                word.getMinXDirAdj(),
-                word.getMaxYDirAdj(),
-                horizontalRulingLines,
-                word.getDir().getDegrees(),
-                word.getPageWidth(),
-                word.getPageHeight()) //
-                || isSplitByRuling(maxX,
-                minY,
-                word.getMinXDirAdj(),
-                word.getMinYDirAdj(),
-                horizontalRulingLines,
-                word.getDir().getDegrees(),
-                word.getPageWidth(),
-                word.getPageHeight()) //
-                || isSplitByRuling(minX,
-                minY,
-                word.getMinXDirAdj(),
-                word.getMaxYDirAdj(),
-                verticalRulingLines,
-                word.getDir().getDegrees(),
-                word.getPageWidth(),
-                word.getPageHeight()); //
-    }
-
-
-    private boolean isSplitByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines, float dir, float pageWidth, float pageHeight) {
-
-        for (Ruling ruling : rulingLines) {
-            var line = RulingTextDirAdjustUtil.convertToDirAdj(ruling, dir, pageWidth, pageHeight);
-            if (line.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-}
-
@@ -1,114 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.classification;
-
-import java.util.List;
-import java.util.regex.Pattern;
-
-import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class TaasClassificationService {
-
-    private final BodyTextFrameService bodyTextFrameService;
-
-
-    public void classifyDocument(ClassificationDocument document) {
-
-
-        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
-
-        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
-
-        for (ClassificationPage page : document.getPages()) {
-
-            classifyPage(page, document, headlineFontSizes);
-        }
-    }
-
-
-    public void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
-
-        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
-            if (textBlock instanceof TextPageBlock) {
-                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
-            }
-        }
-    }
-
-
-    public void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
-
-        var bodyTextFrame = page.getBodyTextFrame();
-
-        if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification(PageBlockType.OTHER);
-            return;
-        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-                || PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
-            textBlock.setClassification(PageBlockType.HEADER);
-        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
-                || PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation())) {
-            textBlock.setClassification(PageBlockType.FOOTER);
-        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
-                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
-                .size() == 1)) {
-            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification(PageBlockType.TITLE);
-            }
-        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                .getCountPerValue()
-                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-
-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                    textBlock.setClassification(PageBlockType.getHeadlineType(i));
-                    document.setHeadlines(true);
-                }
-            }
-        } else if (!textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-            textBlock.setClassification(PageBlockType.getHeadlineType(headlineFontSizes.size() + 1));
-            document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
-                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
-                .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
-            textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
-        } else {
-            textBlock.setClassification(PageBlockType.OTHER);
-        }
-    }
-
-}
@@ -1,10 +1,12 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;

+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
@@ -27,22 +29,31 @@ public class DocstrumSegmentationService {
    private final ReadingOrderService readingOrderService;


-    public List<Zone> segmentPage(List<TextPositionSequence> textPositions) {
+    public List<Zone> segmentPage(List<TextPositionSequence> textPositions, boolean xyOrder) {

-        var positions = textPositions.stream().map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();
+        List<Zone> zones = new ArrayList<>();
+        zones.addAll(computeZones(textPositions, TextDirection.ZERO));
+        zones.addAll(computeZones(textPositions, TextDirection.QUARTER_CIRCLE));
+        zones.addAll(computeZones(textPositions, TextDirection.HALF_CIRCLE));
+        zones.addAll(computeZones(textPositions, TextDirection.THREE_QUARTER_CIRCLE));
+
+        return readingOrderService.resolve(zones, xyOrder);
+    }
+
+
+    private List<Zone> computeZones(List<TextPositionSequence> textPositions, TextDirection direction) {
+
+        var positions = textPositions.stream().filter(t -> t.getDir() == direction).map(TextPositionSequence::getTextPositions).flatMap(List::stream).toList();

        var characters = positions.stream().map(Character::new).collect(Collectors.toList());

        nearestNeighbourService.findNearestNeighbors(characters);

        var characterSpacing = spacingService.computeCharacterSpacing(characters);
-        var lineSpacing = spacingService.computeLineSpacing(characters);
+        var lineSpacing = Math.min(spacingService.computeLineSpacing(characters), 20);

        var lines = lineBuilderService.buildLines(characters, characterSpacing, lineSpacing);
-
-        var zones = zoneBuilderService.buildZones(lines, characterSpacing, lineSpacing);
-
-        return readingOrderService.resolve(zones);
+        return zoneBuilderService.buildZones(lines, characterSpacing, lineSpacing);
    }

 }
@@ -1,90 +1,25 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;

-/**
- * Filter class for neighbor objects that checks if the angle of the
- * neighbor is within specified range.
- */
-public abstract class AngleFilter {
+public class AngleFilter {

-    private final double lowerAngle;
-    private final double upperAngle;
+    protected double lowerAngle;
+    protected double upperAngle;


-    private AngleFilter(double lowerAngle, double upperAngle) {
-
-        this.lowerAngle = lowerAngle;
-        this.upperAngle = upperAngle;
+    public AngleFilter(double lowerAngle, double upperAngle) {
+        
+        this.lowerAngle = lowerAngle < -Math.PI / 2 ? lowerAngle + Math.PI : lowerAngle;
+        this.upperAngle = upperAngle >= Math.PI / 2 ? upperAngle - Math.PI : upperAngle;
    }


-    /**
-     * Constructs new angle filter.
-     *
-     * @param lowerAngle minimum angle in range [-3*pi/2, pi/2)
-     * @param upperAngle maximum angle in range [-pi/2, 3*pi/2)
-     * @return newly constructed angle filter
-     */
-    public static AngleFilter newInstance(double lowerAngle, double upperAngle) {
+    public boolean matches(Neighbor neighbor) {

-        if (lowerAngle < -Math.PI / 2) {
-            lowerAngle += Math.PI;
-        }
-        if (upperAngle >= Math.PI / 2) {
-            upperAngle -= Math.PI;
-        }
        if (lowerAngle <= upperAngle) {
-            return new AndFilter(lowerAngle, upperAngle);
+            return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
        } else {
-            return new OrFilter(lowerAngle, upperAngle);
+            return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
        }
    }

-
-    public double getLowerAngle() {
-
-        return lowerAngle;
-    }
-
-
-    public double getUpperAngle() {
-
-        return upperAngle;
-    }
-
-
-    public abstract boolean matches(Neighbor neighbor);
-
-
-    public static final class AndFilter extends AngleFilter {
-
-        private AndFilter(double lowerAngle, double upperAngle) {
-
-            super(lowerAngle, upperAngle);
-        }
-
-
-        @Override
-        public boolean matches(Neighbor neighbor) {
-
-            return getLowerAngle() <= neighbor.getAngle() && neighbor.getAngle() < getUpperAngle();
-        }
-
-    }
-
-    public static final class OrFilter extends AngleFilter {
-
-        private OrFilter(double lowerAngle, double upperAngle) {
-
-            super(lowerAngle, upperAngle);
-        }
-
-
-        @Override
-        public boolean matches(Neighbor neighbor) {
-
-            return getLowerAngle() <= neighbor.getAngle() || neighbor.getAngle() < getUpperAngle();
-        }
-
-    }
-
 }
@@ -42,7 +42,16 @@ public abstract class BoundingBox {

    public boolean contains(Rectangle2D contained, double tolerance) {

-        return bBox.getX() <= contained.getX() + tolerance && bBox.getY() <= contained.getY() + tolerance && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
+        return bBox.getX() <= contained.getX() + tolerance
+               && bBox.getY() <= contained.getY() + tolerance
+               && bBox.getX() + bBox.getWidth() >= contained.getX() + contained.getWidth() - tolerance
+               && bBox.getY() + bBox.getHeight() >= contained.getY() + contained.getHeight() - tolerance;
+    }
+
+
+    public boolean intersectsY(BoundingBox other) {
+
+        return this.getBBox().getMinY() <= other.getBBox().getMaxY() && this.getBBox().getMaxY() >= other.getBBox().getMinY();
    }

 }
@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;

 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;

 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
@@ -51,6 +52,21 @@ public class Character {
    }


+    public double overlappingDistance(Character other) {
+
+        double[] xs = new double[4];
+        double s = Math.sin(-0);
+        double c = Math.cos(-0);
+        xs[0] = c * x - s * y;
+        xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
+        xs[2] = c * other.x - s * other.y;
+        xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
+        boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
+        Arrays.sort(xs);
+        return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
+    }
+
+
    public void setNeighbors(List<Neighbor> neighbors) {

        this.neighbors = neighbors;
@@ -1,212 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
-
-import java.util.AbstractSet;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-public class DisjointSets<E> implements Iterable<Set<E>> {
-
-    private final Map<E, Entry<E>> map = new HashMap<E, Entry<E>>();
-
-
-    /**
-     * Constructs a new set of singletons.
-     *
-     * @param c elements of singleton sets
-     */
-    public DisjointSets(Collection<? extends E> c) {
-
-        for (E element : c) {
-            map.put(element, new Entry<E>(element));
-        }
-    }
-
-
-    /**
-     * Checks if elements are in the same subsets.
-     *
-     * @param e1 element from a subset
-     * @param e2 element from a subset
-     * @return true if elements are in the same subset; false otherwise
-     */
-    public boolean areTogether(E e1, E e2) {
-
-        return map.get(e1).findRepresentative() == map.get(e2).findRepresentative();
-    }
-
-
-    /**
-     * Merges subsets which elements e1 and e2 belong to.
-     *
-     * @param e1 element from a subset
-     * @param e2 element from a subset
-     */
-    public void union(E e1, E e2) {
-
-        Entry<E> r1 = map.get(e1).findRepresentative();
-        Entry<E> r2 = map.get(e2).findRepresentative();
-        if (r1 != r2) {
-            if (r1.size <= r2.size) {
-                r2.mergeWith(r1);
-            } else {
-                r1.mergeWith(r2);
-            }
-        }
-    }
-
-
-    @Override
-    public Iterator<Set<E>> iterator() {
-
-        return new Iterator<Set<E>>() {
-
-            private final Iterator<Entry<E>> iterator = map.values().iterator();
-            private Entry<E> nextRepresentative;
-
-            {
-                findNextRepresentative();
-            }
-
-            @Override
-            public boolean hasNext() {
-
-                return nextRepresentative != null;
-            }
-
-
-            @Override
-            public Set<E> next() {
-
-                if (nextRepresentative == null) {
-                    throw new NoSuchElementException();
-                }
-                Set<E> result = nextRepresentative.asSet();
-                findNextRepresentative();
-                return result;
-            }
-
-
-            private void findNextRepresentative() {
-
-                while (iterator.hasNext()) {
-                    Entry<E> candidate = iterator.next();
-                    if (candidate.isRepresentative()) {
-                        nextRepresentative = candidate;
-                        return;
-                    }
-                }
-                nextRepresentative = null;
-            }
-
-
-            @Override
-            public void remove() {
-
-                throw new UnsupportedOperationException();
-            }
-
-        };
-    }
-
-
-    private static class Entry<E> {
-
-        private int size = 1;
-        private final E value;
-        private Entry<E> parent = this;
-        private Entry<E> next = null;
-        private Entry<E> last = this;
-
-
-        Entry(E value) {
-
-            this.value = value;
-        }
-
-
-        void mergeWith(Entry<E> otherRepresentative) {
-
-            size += otherRepresentative.size;
-            last.next = otherRepresentative;
-            last = otherRepresentative.last;
-            otherRepresentative.parent = this;
-        }
-
-
-        Entry<E> findRepresentative() {
-
-            Entry<E> representative = parent;
-            while (representative.parent != representative) {
-                representative = representative.parent;
-            }
-            for (Entry<E> entry = this; entry != representative; ) {
-                Entry<E> nextEntry = entry.parent;
-                entry.parent = representative;
-                entry = nextEntry;
-            }
-            return representative;
-        }
-
-
-        boolean isRepresentative() {
-
-            return parent == this;
-        }
-
-
-        Set<E> asSet() {
-
-            return new AbstractSet<E>() {
-
-                @Override
-                public Iterator<E> iterator() {
-
-                    return new Iterator<E>() {
-
-                        private Entry<E> nextEntry = findRepresentative();
-
-
-                        @Override
-                        public boolean hasNext() {
-
-                            return nextEntry != null;
-                        }
-
-
-                        @Override
-                        public E next() {
-
-                            if (nextEntry == null) {
-                                throw new NoSuchElementException();
-                            }
-                            E result = nextEntry.value;
-                            nextEntry = nextEntry.next;
-                            return result;
-                        }
-
-
-                        @Override
-                        public void remove() {
-
-                            throw new UnsupportedOperationException();
-                        }
-
-                    };
-                }
-
-
-                @Override
-                public int size() {
-
-                    return findRepresentative().size;
-                }
-            };
-        }
-
-    }
-
-}
@@ -1,33 +1,19 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;

-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-public class Histogram implements Iterable<Histogram.Bin> {
+public class Histogram {

    private static final double EPSILON = 1.0e-6;
-
    private final double min;
-    private final double delta;
    private final double resolution;
-
    private double[] frequencies;


-    /**
-     * Constructs a new histogram for values in range [minValue, maxValue] with
-     * given resolution.
-     *
-     * @param minValue   - minimum allowed value
-     * @param maxValue   - maximum allowed value
-     * @param resolution - histogram's resolution
-     */
    public Histogram(double minValue, double maxValue, double resolution) {

        this.min = minValue - EPSILON;
-        this.delta = maxValue - minValue + 2 * EPSILON;
+        double delta = maxValue - minValue + 2 * EPSILON;
        int size = Math.max(1, (int) Math.round((maxValue - minValue) / resolution));
-        this.resolution = this.delta / size;
+        this.resolution = delta / size;
        this.frequencies = new double[size];
    }

@@ -47,34 +33,14 @@ public class Histogram implements Iterable<Histogram.Bin> {
    }


-    public void circularKernelSmooth(double[] kernel) {
-
-        double[] newFrequencies = new double[frequencies.length];
-        int shift = (kernel.length - 1) / 2;
-        for (int i = 0; i < frequencies.length; i++) {
-            for (int d = 0; d < kernel.length; d++) {
-                int j = i + d - shift;
-                if (j < 0) {
-                    j += frequencies.length;
-                } else if (j >= frequencies.length) {
-                    j -= frequencies.length;
-                }
-                newFrequencies[i] += kernel[d] * frequencies[j];
-            }
-        }
-        frequencies = newFrequencies;
-    }
-
-
    public double[] createGaussianKernel(double length, double stdDeviation) {

        int r = (int) Math.round(length / resolution) / 2;
-        stdDeviation /= resolution;

        int size = 2 * r + 1;
        double[] kernel = new double[size];
        double sum = 0;
-        double b = 2 * stdDeviation * stdDeviation;
+        double b = 2 * (stdDeviation / resolution) * (stdDeviation / resolution);
        double a = 1 / Math.sqrt(Math.PI * b);
        for (int i = 0; i < size; i++) {
            kernel[i] = a * Math.exp(-(i - r) * (i - r) / b);
@@ -87,45 +53,24 @@ public class Histogram implements Iterable<Histogram.Bin> {
    }


-    public void circularGaussianSmooth(double windowLength, double stdDeviation) {
-
-        circularKernelSmooth(createGaussianKernel(windowLength, stdDeviation));
-    }
-
-
    public void gaussianSmooth(double windowLength, double stdDeviation) {

        kernelSmooth(createGaussianKernel(windowLength, stdDeviation));
    }


-    /**
-     * Adds single occurrence of given value to the histogram.
-     *
-     * @param value inserted values
-     */
    public void add(double value) {

        frequencies[(int) ((value - min) / resolution)] += 1.0;
    }


-    /**
-     * Returns histogram's number of bins.
-     *
-     * @return number of bins
-     */
    public int getSize() {

        return frequencies.length;
    }


-    /**
-     * Finds the histogram's peak value.
-     *
-     * @return peak value
-     */
    public double getPeakValue() {

        int peakIndex = 0;
@@ -142,58 +87,4 @@ public class Histogram implements Iterable<Histogram.Bin> {
        return ((double) peakIndex + peakEndIndex) / 2 * resolution + min;
    }

-
-    @Override
-    public Iterator<Bin> iterator() {
-
-        return new Iterator() {
-
-            private int index = 0;
-
-
-            @Override
-            public boolean hasNext() {
-
-                return index < frequencies.length;
-            }
-
-
-            @Override
-            public Object next() {
-
-                if (index >= frequencies.length) {
-                    throw new NoSuchElementException();
-                }
-                return new Bin(index++);
-            }
-
-
-            @Override
-            public void remove() {
-
-                throw new UnsupportedOperationException("Not supported yet.");
-            }
-
-        };
-    }
-
-
-    public final class Bin {
-
-        private final int index;
-
-
-        private Bin(int index) {
-
-            this.index = index;
-        }
-
-
-        public double getValue() {
-
-            return (index + 0.5) * resolution + min;
-        }
-
-    }
-
 }
@@ -12,7 +12,7 @@ import lombok.Data;
@Data
 public class Line extends BoundingBox {

-    private static final double WORD_DISTANCE_MULTIPLIER = 0.2;
+    private static final double WORD_DISTANCE_MULTIPLIER = 0.18;

    private final double x0;
    private final double y0;
@@ -31,13 +31,16 @@ public class Line extends BoundingBox {
        this.characters = characters;

        if (characters.size() >= 2) {
-            // Simple linear regression
-            double sx = 0.0, sxx = 0.0, sxy = 0.0, sy = 0.0;
-            for (Character component : characters) {
-                sx += component.getX();
-                sxx += component.getX() * component.getX();
-                sxy += component.getX() * component.getY();
-                sy += component.getY();
+            // linear regression
+            double sx = 0.0;
+            double sxx = 0.0;
+            double sxy = 0.0;
+            double sy = 0.0;
+            for (Character character : characters) {
+                sx += character.getX();
+                sxx += character.getX() * character.getX();
+                sxy += character.getX() * character.getY();
+                sy += character.getY();
            }
            double b = (characters.size() * sxy - sx * sy) / (characters.size() * sxx - sx * sx);
            double a = (sy - b * sx) / characters.size();
@@ -46,20 +49,18 @@ public class Line extends BoundingBox {
            this.y0 = a + b * this.x0;
            this.x1 = characters.get(characters.size() - 1).getX();
            this.y1 = a + b * this.x1;
-        } else if (!characters.isEmpty()) {
-            Character component = characters.get(0);
-            double dx = component.getTextPosition().getWidthDirAdj() / 3;
-            double dy = dx * Math.tan(0);
-            this.x0 = component.getX() - dx;
-            this.x1 = component.getX() + dx;
-            this.y0 = component.getY() - dy;
-            this.y1 = component.getY() + dy;
        } else {
-            throw new IllegalArgumentException("Component list must not be empty");
+            Character character = characters.get(0);
+            double dx = character.getTextPosition().getWidthDirAdj() / 3;
+            double dy = dx * Math.tan(0);
+            this.x0 = character.getX() - dx;
+            this.x1 = character.getX() + dx;
+            this.y0 = character.getY() - dy;
+            this.y1 = character.getY() + dy;
        }
        height = computeHeight();
        computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
-        buildBox();
+        buildBBox();
    }


@@ -77,11 +78,7 @@ public class Line extends BoundingBox {

    private double computeHeight() {

-        double sum = 0.0;
-        for (Character component : characters) {
-            sum += component.getHeight();
-        }
-        return sum / characters.size();
+        return characters.stream().map(Character::getHeight).reduce(0d, Double::sum) / characters.size();
    }


@@ -136,7 +133,7 @@ public class Line extends BoundingBox {
    }


-    private void buildBox() {
+    private void buildBBox() {

        double minX = Double.POSITIVE_INFINITY;
        double minY = Double.POSITIVE_INFINITY;
@@ -0,0 +1,31 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class UnionFind<T> extends org.jgrapht.alg.util.UnionFind<T> {
+
+    public UnionFind(Set<T> elements) {
+
+        super(elements);
+    }
+
+
+    public Collection<Set<T>> getGroups() {
+
+        Map<T, Set<T>> setRep = new LinkedHashMap<>();
+        for (T t : getParentMap().keySet()) {
+            T representative = find(t);
+            if (!setRep.containsKey(representative)) {
+                setRep.put(representative, new LinkedHashSet<>());
+            }
+            setRep.get(representative).add(t);
+        }
+
+        return setRep.values();
+    }
+
+}
@@ -12,15 +12,16 @@ public class Zone extends BoundingBox {
    private List<Line> lines;


+    @SuppressWarnings("PMD.ConstructorCallsOverridableMethod")
    public Zone(List<Line> lines) {

        lines.sort(Comparator.comparingDouble(Line::getY));
        this.lines = lines;
-        buildBox();
+        buildBBox();
    }


-    public void buildBox() {
+    public void buildBBox() {

        double minX = Double.POSITIVE_INFINITY;
        double minY = Double.POSITIVE_INFINITY;
@@ -1,64 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder;
-
-import java.awt.geom.Rectangle2D;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
-
-public class BoundingBoxZoneGroup extends BoundingBox {
-
-    private BoundingBox leftChild;
-    private BoundingBox rightChild;
-
-
-    public BoundingBoxZoneGroup(BoundingBox child1, BoundingBox child2) {
-
-        this.leftChild = child1;
-        this.rightChild = child2;
-        setBounds(Math.min(child1.getX(), child2.getX()),
-                Math.min(child1.getY(), child2.getY()),
-                Math.max(child1.getX() + child1.getWidth(), child2.getX() + child2.getWidth()),
-                Math.max(child1.getY() + child1.getHeight(), child2.getY() + child2.getHeight()));
-    }
-
-
-    public void setbBox(Rectangle2D bBox) {
-
-        super.setBBox(bBox);
-    }
-
-
-    public BoundingBox getLeftChild() {
-
-        return leftChild;
-    }
-
-
-    public BoundingBox getRightChild() {
-
-        return rightChild;
-    }
-
-
-    public BoundingBoxZoneGroup setLeftChild(BoundingBox obj) {
-
-        this.leftChild = obj;
-        return this;
-    }
-
-
-    public BoundingBoxZoneGroup setRightChild(BoundingBox obj) {
-
-        this.rightChild = obj;
-        return this;
-    }
-
-
-    public BoundingBoxZoneGroup setBounds(double x0, double y0, double x1, double y1) {
-
-        assert x1 >= x0;
-        assert y1 >= y0;
-        this.setBBox(new Rectangle2D.Double(x0, y0, x1 - x0, y1 - y0));
-        return this;
-    }
-
-}
@@ -1,115 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
-
-public class DistElem<E> implements Comparable<DistElem<E>> {
-
-    @Override
-    public int hashCode() {
-
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + (c ? 1231 : 1237);
-        long temp;
-        temp = Double.doubleToLongBits(dist);
-        result = prime * result + (int) (temp ^ (temp >>> 32));
-        result = prime * result + ((obj1 == null) ? 0 : obj1.hashCode());
-        result = prime * result + ((obj2 == null) ? 0 : obj2.hashCode());
-        return result;
-    }
-
-
-    @Override
-    public boolean equals(Object obj) {
-
-        if (this == obj) {
-            return true;
-        }
-        if (obj == null) {
-            return false;
-        }
-        if (getClass() != obj.getClass()) {
-            return false;
-        }
-        DistElem other = (DistElem) obj;
-        if (c != other.c) {
-            return false;
-        }
-        if (Double.doubleToLongBits(dist) != Double.doubleToLongBits(other.dist)) {
-            return false;
-        }
-        if (obj1 == null) {
-            if (other.obj1 != null) {
-                return false;
-            }
-        } else if (!obj1.equals(other.obj1)) {
-            return false;
-        }
-        if (obj2 == null) {
-            if (other.obj2 != null) {
-                return false;
-            }
-        } else if (!obj2.equals(other.obj2)) {
-            return false;
-        }
-        return true;
-    }
-
-
-    boolean c;
-    double dist;
-    E obj1;
-    E obj2;
-
-
-    public boolean isC() {
-
-        return c;
-    }
-
-
-    public void setC(boolean c) {
-
-        this.c = c;
-    }
-
-
-    public double getDist() {
-
-        return dist;
-    }
-
-
-    public E getObj1() {
-
-        return obj1;
-    }
-
-
-    public E getObj2() {
-
-        return obj2;
-    }
-
-
-    public DistElem(boolean c, double dist, E obj1, E obj2) {
-
-        this.c = c;
-        this.dist = dist;
-        this.obj1 = obj1;
-        this.obj2 = obj2;
-    }
-
-
-    @Override
-    public int compareTo(DistElem<E> compareObject) {
-
-        double eps = 1E-3;
-        if (c == compareObject.c) {
-            return DoubleUtils.compareDouble(dist, compareObject.dist, eps);
-        } else {
-            return c ? -1 : 1;
-        }
-    }
-
-}
@@ -1,258 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-
-/**
- * A set-like data structure for objects placed on a plane. Can efficiently find objects in a certain rectangular area.
- * It maintains two parallel lists of objects, each of which is sorted by its x or y coordinate.
- *
- * @author Pawel Szostek
- */
-public class DocumentPlane {
-
-    /**
-     * List of objects on the plane. Stored in a random order
-     */
-    private final List<BoundingBox> objs;
-    /**
-     * Size of a grid square. If gridSize=50, then the plane is divided into squares of size 50. Each square contains
-     * objects placed in a 50x50 area
-     */
-    private final int gridSize;
-    /**
-     * Redundant dictionary of objects on the plane. Allows efficient 2D space search. Keys are X-Y coordinates of a
-     * grid square. Single object can be stored under several keys (depending on its physical size). Grid squares are
-     * lazy-initialized.
-     */
-    private final Map<GridXY, List<BoundingBox>> grid;
-
-    /**
-     * Representation of XY coordinates
-     */
-    private static class GridXY {
-
-        public int x;
-        public int y;
-
-
-        public GridXY(int x, int y) {
-
-            this.x = x;
-            this.y = y;
-        }
-
-
-        @Override
-        public int hashCode() {
-
-            return x * y;
-        }
-
-
-        @Override
-        public boolean equals(Object obj) {
-
-            if (obj == null || getClass() != obj.getClass()) {
-                return false;
-            }
-            GridXY comparedObj = (GridXY) obj;
-            return x == comparedObj.x && y == comparedObj.y;
-        }
-
-
-        @Override
-        public String toString() {
-
-            return "(" + x + "," + y + ")";
-        }
-
-    }
-
-
-    public List<BoundingBox> getObjects() {
-
-        return objs;
-    }
-
-
-    public DocumentPlane(List<Zone> objectList, int gridSize) {
-
-        this.grid = new HashMap<GridXY, List<BoundingBox>>();
-        this.objs = new ArrayList<BoundingBox>();
-        this.gridSize = gridSize;
-        for (Zone obj : objectList) {
-            add(obj);
-        }
-    }
-
-
-    /**
-     * Looks for objects placed between obj1 and obj2 excluding them
-     *
-     * @param obj1 object
-     * @param obj2 object
-     * @return object list
-     */
-    public List<BoundingBox> findObjectsBetween(BoundingBox obj1, BoundingBox obj2) {
-
-        double x0 = Math.min(obj1.getX(), obj2.getX());
-        double y0 = Math.min(obj1.getY(), obj2.getY());
-        double x1 = Math.max(obj1.getX() + obj1.getWidth(), obj2.getX() + obj2.getWidth());
-        double y1 = Math.max(obj1.getY() + obj1.getHeight(), obj2.getY() + obj2.getHeight());
-        assert x1 >= x0 && y1 >= y0;
-        Rectangle2D searchBounds = new Rectangle2D.Double(x0, y0, x1 - x0, y1 - y0);
-        List<BoundingBox> objsBetween = find(searchBounds);
-        /*
-         * the rectangle area must contain at least obj1 and obj2
-         */
-        objsBetween.remove(obj1);
-        objsBetween.remove(obj2);
-        return objsBetween;
-    }
-
-
-    /**
-     * Checks if there is any object placed between obj1 and obj2
-     *
-     * @param obj1 object
-     * @param obj2 object
-     * @return true if anything is placed between, false otherwise
-     */
-    public boolean anyObjectsBetween(BoundingBox obj1, BoundingBox obj2) {
-
-        List<BoundingBox> lObjs = findObjectsBetween(obj1, obj2);
-        return !(lObjs.isEmpty());
-    }
-
-
-    /**
-     * Adds object to the plane
-     *
-     * @param obj object
-     * @return document plane
-     */
-    public DocumentPlane add(BoundingBox obj) {
-
-        int objsBefore = this.objs.size();
-        /*
-         * iterate over grid squares
-         */
-        for (int y = ((int) obj.getY()) / gridSize; y <= ((int) (obj.getY() + obj.getHeight() + gridSize - 1)) / gridSize; ++y) {
-            for (int x = ((int) obj.getX()) / gridSize; x <= ((int) (obj.getX() + obj.getWidth() + gridSize - 1)) / gridSize; ++x) {
-                GridXY xy = new GridXY(x, y);
-                if (!grid.keySet().contains(xy)) {
-                    /*
-                     * add the non-existing key
-                     */
-                    grid.put(xy, new ArrayList<BoundingBox>());
-                    grid.get(xy).add(obj);
-                    assert grid.get(xy).size() == 1;
-                } else {
-                    grid.get(xy).add(obj);
-                }
-            }
-        }
-        objs.add(obj);
-        /*
-         * size of the object list should be incremented
-         */
-        assert objsBefore + 1 == objs.size();
-        /*
-         * object list must contain the same number of objects as object dictionary
-         */
-        assert objs.size() == elementsInGrid();
-        return this;
-    }
-
-
-    public DocumentPlane remove(BoundingBox obj) {
-        /*
-         * iterate over grid squares
-         */
-        for (int y = ((int) obj.getY()) / gridSize; y <= ((int) (obj.getY() + obj.getHeight() + gridSize - 1)) / gridSize; ++y) {
-            for (int x = ((int) obj.getX()) / gridSize; x <= ((int) (obj.getX() + obj.getWidth() + gridSize - 1)) / gridSize; ++x) {
-                GridXY xy = new GridXY(x, y);
-                if (grid.get(xy).contains(obj)) {
-                    grid.get(xy).remove(obj);
-                }
-            }
-        }
-        objs.remove(obj);
-        assert objs.size() == elementsInGrid();
-        return this;
-    }
-
-
-    /**
-     * Find objects within search bounds
-     *
-     * @param searchBounds is a search rectangle
-     * @return list of objects in!side search rectangle
-     */
-    public List<BoundingBox> find(Rectangle2D searchBounds) {
-
-        List<BoundingBox> done = new ArrayList<BoundingBox>(); //contains already considered objects (wrt. optimization)
-        List<BoundingBox> ret = new ArrayList<BoundingBox>();
-        double x0 = searchBounds.getX();
-        double y0 = searchBounds.getY();
-        double y1 = searchBounds.getY() + searchBounds.getHeight();
-        double x1 = searchBounds.getX() + searchBounds.getWidth();
-        /*
-         * iterate over grid squares
-         */
-        for (int y = (int) y0 / gridSize; y < ((int) (y1 + gridSize - 1)) / gridSize; ++y) {
-            for (int x = (int) x0 / gridSize; x < ((int) (x1 + gridSize - 1)) / gridSize; ++x) {
-                GridXY xy = new GridXY(x, y);
-                if (!grid.containsKey(xy)) {
-                    continue;
-                }
-                for (BoundingBox obj : grid.get(xy)) {
-                    if (done.contains(obj)) /*
-                     * omit if already checked
-                     */ {
-                        continue;
-                    }
-                    /*
-                     * add to the checked objects
-                     */
-                    done.add(obj);
-                    /*
-                     * check if two objects overlap
-                     */
-                    if (obj.getX() + obj.getWidth() <= x0 || x1 <= obj.getX() || obj.getY() + obj.getHeight() <= y0 || y1 <= obj.getY()) {
-                        continue;
-                    }
-                    ret.add(obj);
-                }
-            }
-        }
-        return ret;
-    }
-
-
-    /**
-     * Count objects stored in objects dictionary
-     *
-     * @return number of elements
-     */
-    protected int elementsInGrid() {
-
-        List<BoundingBox> objs_ = new ArrayList<BoundingBox>();
-        for (GridXY coord : grid.keySet()) {
-            for (BoundingBox obj : grid.get(coord)) {
-                if (!objs_.contains(obj)) {
-                    objs_.add(obj);
-                }
-            }
-        }
-        return objs_.size();
-    }
-
-}
@@ -1,29 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-
-public class TreeToListConverter {
-
-    public List<Zone> convertToList(BoundingBoxZoneGroup obj) {
-
-        List<Zone> ret = new ArrayList<>();
-        if (obj.getLeftChild() instanceof Zone) {
-            Zone zone = (Zone) obj.getLeftChild();
-            ret.add(zone);
-        } else { // obj.getLeftChild() instanceof BxZoneGroup
-            ret.addAll(convertToList((BoundingBoxZoneGroup) obj.getLeftChild()));
-        }
-
-        if (obj.getRightChild() instanceof Zone) {
-            Zone zone = (Zone) obj.getRightChild();
-            ret.add(zone);
-        } else { // obj.getRightChild() instanceof BxZoneGroup
-            ret.addAll(convertToList((BoundingBoxZoneGroup) obj.getRightChild()));
-        }
-        return ret;
-    }
-
-}
@@ -2,14 +2,15 @@ package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.s

 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;

 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.AngleFilter;
 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.UnionFind;

@Service
 public class LineBuilderService {
@@ -24,24 +25,26 @@ public class LineBuilderService {
        double maxHorizontalDistance = characterSpacing * CHARACTER_SPACING_DISTANCE_MULTIPLIER;
        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_CHARACTER_DISTANCE;

-        DisjointSets<Character> sets = new DisjointSets<>(characters);
-        AngleFilter filter = AngleFilter.newInstance(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
+        UnionFind<Character> unionFind = new UnionFind<>(new HashSet<>(characters));
+
+        AngleFilter filter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);

        characters.forEach(character -> {
            character.getNeighbors().forEach(neighbor -> {
                double x = neighbor.getHorizontalDistance() / maxHorizontalDistance;
                double y = neighbor.getVerticalDistance() / maxVerticalDistance;
-                if (filter.matches(neighbor) && Math.pow(x, 2) + Math.pow(y, 2) <= 1) {
-                    sets.union(character, neighbor.getCharacter());
+                if (character.getTextPosition().getDir() == neighbor.getCharacter().getTextPosition().getDir() && filter.matches(neighbor) && Math.pow(x, 2) + Math.pow(y,
+                        2) <= 1) {
+                    unionFind.union(character, neighbor.getCharacter());
                }
            });
        });

        List<Line> lines = new ArrayList<>();
-        sets.forEach(group -> {
-            List<Character> lineComponents = new ArrayList<>(group);
-            lineComponents.sort(Comparator.comparingDouble(Character::getX));
-            lines.add(new Line(lineComponents, characterSpacing));
+        unionFind.getGroups().forEach(group -> {
+            List<Character> lineCharacters = new ArrayList<>(group);
+            lineCharacters.sort(Comparator.comparingDouble(Character::getX));
+            lines.add(new Line(lineCharacters, characterSpacing));
        });

        return lines;
@@ -18,7 +18,7 @@ public class NearestNeighbourService {

    public void findNearestNeighbors(List<Character> characters) {

-        if (characters.isEmpty()) {
+        if (characters.isEmpty() || characters.size() == 1) {
            return;
        }

@@ -1,286 +1,165 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;

 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;

 import org.springframework.stereotype.Service;

 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.BoundingBoxZoneGroup;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.DistElem;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.DocumentPlane;
-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.TreeToListConverter;
 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;

@Service
 public class ReadingOrderService {

-    static final int GRIDSIZE = 50;
-    static final double EPS = 0.01;
-    static final int MAX_ZONES = 1000;
-    static final Comparator<BoundingBox> Y_ASCENDING_ORDER = new Comparator<BoundingBox>() {
+    private static final double THRESHOLD = 5;
+    public static final double MULTI_COLUMN_DETECTION_THRESHOLD = 1.5;

-        @Override
-        public int compare(BoundingBox o1, BoundingBox o2) {

-            return DoubleUtils.compareDouble(o1.getY(), o2.getY(), EPS);
+    public List<Zone> resolve(List<Zone> zones, boolean xyReadingOrder) {
+
+        if (zones.isEmpty() || zones.size() == 1) {
+            return zones;
        }
-    };

-    static final Comparator<BoundingBox> X_ASCENDING_ORDER = new Comparator<BoundingBox>() {
-
-        @Override
-        public int compare(BoundingBox o1, BoundingBox o2) {
-
-            return DoubleUtils.compareDouble(o1.getX(), o2.getX(), EPS);
+        if (xyReadingOrder) {
+            return resolveSingleColumnReadingOrder(zones);
        }
-    };

-    static final Comparator<BoundingBox> YX_ASCENDING_ORDER = new Comparator<BoundingBox>() {
-
-        @Override
-        public int compare(BoundingBox o1, BoundingBox o2) {
-
-            int yCompare = Y_ASCENDING_ORDER.compare(o1, o2);
-            return yCompare == 0 ? X_ASCENDING_ORDER.compare(o1, o2) : yCompare;
+        Map<Long, Integer> histogram = new HashMap<>();
+        for (Zone zone : zones) {
+            long minY = Math.round(zone.getBBox().getMinY());
+            long maxY = Math.round(zone.getBBox().getMaxY());
+            for (long i = minY; i <= maxY; i++) {
+                histogram.put(i, histogram.getOrDefault(i, 0) + 1);
+            }
        }
-    };

-
-    public List<Zone> resolve(List<Zone> zones) {
-
-        List<Zone> orderedZones;
-        if (zones.size() > MAX_ZONES) {
-            orderedZones = new ArrayList<>(zones);
-            Collections.sort(orderedZones, YX_ASCENDING_ORDER);
+        if (histogram.values().stream().mapToInt(Integer::intValue).average().orElse(1) < MULTI_COLUMN_DETECTION_THRESHOLD) {
+            return resolveSingleColumnReadingOrder(zones);
        } else {
-            orderedZones = reorderZones(zones);
+
+            return resolveMultiColumnReadingOder(zones);
        }
-        return orderedZones;
+
    }


-    private List<Zone> reorderZones(List<Zone> unorderedZones) {
+    private static List<Zone> resolveSingleColumnReadingOrder(List<Zone> zones) {

-        if (unorderedZones.isEmpty()) {
-            return new ArrayList<>();
-        } else if (unorderedZones.size() == 1) {
-            List<Zone> ret = new ArrayList<>(1);
-            ret.add(unorderedZones.get(0));
-            return ret;
-        } else {
-            BoundingBoxZoneGroup bxZonesTree = groupZonesHierarchically(unorderedZones);
-            sortGroupedZones(bxZonesTree);
-            TreeToListConverter treeConverter = new TreeToListConverter();
-            List<Zone> orderedZones = treeConverter.convertToList(bxZonesTree);
-            assert unorderedZones.size() == orderedZones.size();
-            return orderedZones;
-        }
+        zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+        return zones;
    }


-    /**
-     * Builds a binary tree of zones and groups of zones from a list of unordered zones. This is done in hierarchical
-     * clustering by joining two least distant nodes. Distance is calculated in the distance() method.
-     *
-     * @param zones is a list of unordered zones
-     * @return root of the zones clustered in a tree
-     */
-    private BoundingBoxZoneGroup groupZonesHierarchically(List<Zone> zones) {
-        /*
-         * Distance tuples are stored sorted by ascending distance value
-         */
-        List<DistElem<BoundingBox>> dists = new ArrayList<DistElem<BoundingBox>>(zones.size() * zones.size() / 2);
-        for (int idx1 = 0; idx1 < zones.size(); ++idx1) {
-            for (int idx2 = idx1 + 1; idx2 < zones.size(); ++idx2) {
-                Zone zone1 = zones.get(idx1);
-                Zone zone2 = zones.get(idx2);
-                dists.add(new DistElem<BoundingBox>(false, distance(zone1, zone2), zone1, zone2));
+    private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
+
+        // Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
+        // TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
+
+        double minX = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+
+        for (Zone zone : zones) {
+            if (zone.getX() < minX) {
+                minX = zone.getX();
+            }
+            if (zone.getX() + zone.getWidth() > maxX) {
+                maxX = zone.getX() + zone.getWidth();
            }
        }
-        Collections.sort(dists);
-        DocumentPlane plane = new DocumentPlane(zones, GRIDSIZE);
-        while (!dists.isEmpty()) {
-            DistElem<BoundingBox> distElem = dists.get(0);
-            dists.remove(0);
-            if (!distElem.isC() && plane.anyObjectsBetween(distElem.getObj1(), distElem.getObj2())) {
-                dists.add(new DistElem<BoundingBox>(true, distElem.getDist(), distElem.getObj1(), distElem.getObj2()));
-                continue;
-            }
-            BoundingBoxZoneGroup newGroup = new BoundingBoxZoneGroup(distElem.getObj1(), distElem.getObj2());
-            plane.remove(distElem.getObj1()).remove(distElem.getObj2());
-            dists = removeDistElementsContainingObject(dists, distElem.getObj1());
-            dists = removeDistElementsContainingObject(dists, distElem.getObj2());
-            for (BoundingBox other : plane.getObjects()) {
-                dists.add(new DistElem<BoundingBox>(false, distance(other, newGroup), newGroup, other));
-            }
-            Collections.sort(dists);
-            plane.add(newGroup);
-        }

-        assert plane.getObjects().size() == 1 : "There should be one object left at the plane after grouping";
-        return (BoundingBoxZoneGroup) plane.getObjects().get(0);
-    }
+        double midLineXCoordinate = (minX + maxX) / 2;

-
-    /**
-     * Removes all distance tuples containing obj
-     */
-    private List<DistElem<BoundingBox>> removeDistElementsContainingObject(Collection<DistElem<BoundingBox>> list, BoundingBox obj) {
-
-        List<DistElem<BoundingBox>> ret = new ArrayList<DistElem<BoundingBox>>();
-        for (DistElem<BoundingBox> distElem : list) {
-            if (distElem.getObj1() != obj && distElem.getObj2() != obj) {
-                ret.add(distElem);
+        List<Zone> leftOf = new ArrayList<>();
+        List<Zone> rightOf = new ArrayList<>();
+        List<Zone> middle = new ArrayList<>();
+        for (Zone zone : zones) {
+            if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
+                leftOf.add(zone);
+            } else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
+                rightOf.add(zone);
+            } else {
+                middle.add(zone);
            }
        }
-        return ret;
-    }

+        leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));

-    /**
-     * Swaps children of BxZoneGroup if necessary. A group with smaller sort factor is placed to the left (leftChild).
-     * An object with greater sort factor is placed on the right (rightChild). This plays an important role when
-     * traversing the tree in conversion to a one dimensional list.
-     *
-     * @param group
-     */
-    private void sortGroupedZones(BoundingBoxZoneGroup group) {
+        rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));

-        BoundingBox leftChild = group.getLeftChild();
-        BoundingBox rightChild = group.getRightChild();
-        if (shouldBeSwapped(leftChild, rightChild)) {
-            // swap
-            group.setLeftChild(rightChild);
-            group.setRightChild(leftChild);
+        middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
+                .thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
+
+        List<Zone> leftNotIntersecting = new ArrayList<>();
+        for (Zone leftZone : leftOf) {
+            boolean intersects = false;
+            for (Zone rightZone : rightOf) {
+                if (leftZone.intersectsY(rightZone)) {
+                    intersects = true;
+                    break;
+                }
+                // early stopping
+                if (rightZone.getBBox().getMinY() > leftZone.getBBox().getMaxY()) {
+                    break;
+                }
+            }
+            if (!intersects) {
+                leftNotIntersecting.add(leftZone);
+            }
        }

-        if (leftChild instanceof BoundingBoxZoneGroup) // if the child is a tree node, then recurse
-        {
-            sortGroupedZones((BoundingBoxZoneGroup) leftChild);
+        List<Zone> rightNotIntersecting = new ArrayList<>();
+        for (Zone rightZone : rightOf) {
+            boolean intersects = false;
+            for (Zone leftZone : leftOf) {
+                if (rightZone.intersectsY(leftZone)) {
+                    intersects = true;
+                    break;
+                }
+                // early stopping
+                if (leftZone.getBBox().getMinY() > rightZone.getBBox().getMaxY()) {
+                    break;
+                }
+            }
+            if (!intersects) {
+                rightNotIntersecting.add(rightZone);
+            }
        }
-        if (rightChild instanceof BoundingBoxZoneGroup) // as above - recurse
-        {
-            sortGroupedZones((BoundingBoxZoneGroup) rightChild);
+
+        leftOf.removeAll(leftNotIntersecting);
+        rightOf.removeAll(rightNotIntersecting);
+
+        middle.addAll(leftNotIntersecting);
+        middle.addAll(rightNotIntersecting);
+
+        List<Zone> sortedZones = new ArrayList<>();
+        sortedZones.addAll(leftOf);
+        sortedZones.addAll(rightOf);
+
+        ListIterator<Zone> itty = middle.listIterator();
+
+        while (itty.hasNext()) {
+            Zone current = itty.next();
+            for (int i = 0; i < sortedZones.size(); i++) {
+                if (current.getY() < sortedZones.get(i).getY()) {
+                    sortedZones.add(i, current);
+                    itty.remove();
+                    break;
+                }
+            }
        }
-    }

+        sortedZones.addAll(middle);

-    private boolean shouldBeSwapped(BoundingBox first, BoundingBox second) {
-
-        double cx, cy, cw, ch, ox, oy, ow, oh;
-        cx = first.getBBox().getX();
-        cy = first.getBBox().getY();
-        cw = first.getBBox().getWidth();
-        ch = first.getBBox().getHeight();
-
-        ox = second.getBBox().getX();
-        oy = second.getBBox().getY();
-        ow = second.getBBox().getWidth();
-        oh = second.getBBox().getHeight();
-
-        // Determine Octant
-        //
-        // 0 | 1 | 2
-        // __|___|__
-        // 7 | 9 | 3   First is placed in 9th square
-        // __|___|__
-        // 6 | 5 | 4
-
-        if (cx + cw <= ox) { //2,3,4
-            return false;
-        } else if (ox + ow <= cx) { //0,6,7
-            return true; //6
-        } else if (cy + ch <= oy) {
-            return false; //5
-        } else if (oy + oh <= cy) {
-            return true; //1
-        } else { //two zones
-            double xdiff = ox + ow / 2 - cx - cw / 2;
-            double ydiff = oy + oh / 2 - cy - ch / 2;
-            return xdiff + ydiff < 0;
-        }
-    }
-
-
-    /**
-     * A distance function between two TextBoxes.
-     * <p>
-     * Consider the bounding rectangle for obj1 and obj2. Return its area minus the areas of obj1 and obj2, shown as
-     * 'www' below. This value may be negative. (x0,y0) +------+..........+ | obj1 |wwwwwwwwww: +------+www+------+
-     * :wwwwwwwwww| obj2 | +..........+------+ (x1,y1)
-     *
-     * @return distance value based on objects' coordinates and physical size on a plane
-     */
-    private double distance(BoundingBox obj1, BoundingBox obj2) {
-
-        double x0 = Math.min(obj1.getX(), obj2.getX());
-        double y0 = Math.min(obj1.getY(), obj2.getY());
-        double x1 = Math.max(obj1.getX() + obj1.getWidth(), obj2.getX() + obj2.getWidth());
-        double y1 = Math.max(obj1.getY() + obj1.getHeight(), obj2.getY() + obj2.getHeight());
-        double dist = ((x1 - x0) * (y1 - y0) - obj1.getArea() - obj2.getArea());
-
-        double factor = ((x1 - x0)/x1) / ((y1 - y0)/y1);
-
-        double obj1X = obj1.getX();
-        double obj1Y_2 = obj1.getBBox().getMaxY();
-        double obj1X_2 = obj1.getBBox().getMaxX();
-        double obj1CenterX = obj1.getBBox().getCenterX();
-        double obj1CenterY = obj1.getBBox().getCenterY();
-        double obj2X = obj2.getX();
-        double obj2Y_2 = obj2.getBBox().getMaxY();
-        double obj2X_2 = obj2.getBBox().getMaxX();
-        double obj2CenterX = obj2.getBBox().getCenterX();
-        double obj2CenterY = obj2.getBBox().getCenterY();
-
-        double obj1obj2VectorCosineAbsLeft = Math.abs((obj2X - obj1X) / Math.sqrt((obj2X - obj1X) * (obj2X - obj1X) + (obj2CenterY - obj1CenterY) * (obj2CenterY - obj1CenterY)));
-        double obj1obj2VectorCosineAbsRight = Math.abs((obj2X_2 - obj1X_2) / Math.sqrt((obj2X_2 - obj1X_2) * (obj2X_2 - obj1X_2) + (obj2CenterY - obj1CenterY) * (obj2CenterY - obj1CenterY)));
-        double obj1obj2VectorCosineAbsCenter = Math.abs((obj2CenterX - obj1CenterX) / Math.sqrt((obj2CenterX - obj1CenterX) * (obj2CenterX - obj1CenterX) + (obj2CenterY - obj1CenterY) * (obj2CenterY - obj1CenterY)));
-
-        double cosine = Math.min(obj1obj2VectorCosineAbsLeft, Math.min(obj1obj2VectorCosineAbsRight, obj1obj2VectorCosineAbsCenter));
-
-        final double MAGIC_COEFF = 0.85;
-        //return dist * (MAGIC_COEFF + cosine);
-
-        return Math.sqrt(Math.pow((obj1X - obj2X), 2) + Math.pow((obj1Y_2 - obj2Y_2) * MAGIC_COEFF, 2));
-
-
-        /**if (Math.abs(obj1CenterX - obj2CenterX) >= Math.abs(obj1CenterY - obj2CenterY)) {
-            return dist * 2;
-        } else {
-            return dist;
-        }**/
-
-
-    }
-
-
-    private double distanceNew(BoundingBox obj1, BoundingBox obj2) {
-
-        if(obj1.getBBox().intersects(obj2.getBBox()))
-            return -1;
-
-        double minX0 = Math.min(obj1.getX(), obj2.getX());
-        double maxX0 = Math.max(obj1.getX(), obj2.getX());
-        double minY0 = Math.min(obj1.getY(), obj2.getY());
-        double maxY0 = Math.max(obj1.getY(), obj2.getY());
-        double minX1 = Math.min(obj1.getX() + obj1.getWidth(), obj2.getX() + obj2.getWidth());
-        double maxX1 = Math.max(obj1.getX() + obj1.getWidth(), obj2.getX() + obj2.getWidth());
-        double minY1 = Math.min(obj1.getY() + obj1.getHeight(), obj2.getY() + obj2.getHeight());
-        double maxY1 = Math.max(obj1.getY() + obj1.getHeight(), obj2.getY() + obj2.getHeight());
-        List<Double> xValues = new ArrayList<>(List.of(minX0, maxX0, minX1, maxX1));
-        Collections.sort(xValues);
-        List<Double> yValues = new ArrayList<>(List.of(minY0, maxY0, minY1, maxY1));
-        Collections.sort(yValues);
-
-        double yArea = (xValues.get(2) - xValues.get(1)) * (yValues.get(3) - yValues.get(0));
-        double xArea = (yValues.get(2) - yValues.get(1)) * (xValues.get(3) - xValues.get(0));
-
-        return Math.min(10*yArea, xArea);
+        return sortedZones;
    }

 }
@@ -18,32 +18,32 @@ public class SpacingService {
    private static final double ANGLE_TOLERANCE = Math.PI / 6;


-    public double computeCharacterSpacing(List<Character> components) {
+    public double computeCharacterSpacing(List<Character> characters) {

-        return computeSpacing(components, 0);
+        return computeSpacing(characters, 0);
    }


-    public double computeLineSpacing(List<Character> components) {
+    public double computeLineSpacing(List<Character> characters) {

-        return computeSpacing(components, Math.PI / 2);
+        return computeSpacing(characters, Math.PI / 2);
    }


-    private double computeSpacing(List<Character> components, double angle) {
+    private double computeSpacing(List<Character> characters, double angle) {

        double maxDistance = Double.NEGATIVE_INFINITY;

-        for (Character component : components) {
-            for (Neighbor neighbor : component.getNeighbors()) {
+        for (Character character : characters) {
+            for (Neighbor neighbor : character.getNeighbors()) {
                maxDistance = Math.max(maxDistance, neighbor.getDistance());
            }
        }
        Histogram histogram = new Histogram(0, maxDistance, SPACING_HISTOGRAM_RESOLUTION);
-        AngleFilter filter = AngleFilter.newInstance(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
-        for (Character component : components) {
-            for (Neighbor neighbor : component.getNeighbors()) {
-                if (filter.matches(neighbor)) {
+        AngleFilter angleFilter = new AngleFilter(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
+        for (Character character : characters) {
+            for (Neighbor neighbor : character.getNeighbors()) {
+                if (angleFilter.matches(neighbor)) {
                    histogram.add(neighbor.getDistance());
                }
            }
@@ -1,12 +1,16 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;

 import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;

 import org.springframework.stereotype.Service;

-import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
+import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.UnionFind;
 import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;

@Service
@@ -25,7 +29,9 @@ public class ZoneBuilderService {

    private static final double ANGLE_TOLERANCE = Math.PI / 6;

-    public static final int MAX_ZONES = 300;
+    private static final int MAX_ZONES = 300;
+
+    private static final double MAX_VERTICAL_MERGE_DISTANCE = 0.5;


    public List<Zone> buildZones(List<Line> lines, double characterSpacing, double lineSpacing) {
@@ -35,34 +41,30 @@ public class ZoneBuilderService {
        double minHorizontalMergeDistance = characterSpacing * MIN_HORIZONTAL_MERGE_DISTANCE_MULTIPLIER;
        double maxVerticalMergeDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE_MULTIPLIER;

-        DisjointSets<Line> sets = new DisjointSets<>(lines);
+        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));

        double meanHeight = calculateMeanHeight(lines);

        lines.forEach(outerLine -> //
                lines.forEach(innerLine -> {
+
                    double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
                    scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));

-                    if (!sets.areTogether(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {
+                    if (!unionFind.inSameSet(outerLine, innerLine) && outerLine.angularDifference(innerLine) <= ANGLE_TOLERANCE) {

                        double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
                        double verticalDistance = outerLine.verticalDistance(innerLine) / scale;

-                        // Line over or above
-                        if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance) {
-                            sets.union(outerLine, innerLine);
-                        }
-
-                        // Split line that needs later merging
-                        else if (minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
-                            sets.union(outerLine, innerLine);
+                        if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance //
+                                || minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
+                            unionFind.union(outerLine, innerLine);
                        }
                    }
                }));

        List<Zone> zones = new ArrayList<>();
-        sets.forEach(group -> {
+        unionFind.getGroups().forEach(group -> {
            zones.add(new Zone(new ArrayList<>(group)));
        });

@@ -71,7 +73,7 @@ public class ZoneBuilderService {
            for (Zone zone : zones) {
                oneZoneLines.addAll(zone.getLines());
            }
-            return List.of(new Zone(oneZoneLines));
+            return List.of(mergeLinesInZone(oneZoneLines, characterSpacing, lineSpacing));
        }

        return zones;
@@ -91,4 +93,60 @@ public class ZoneBuilderService {
        return meanHeight;
    }

+
+    private Zone mergeLinesInZone(List<Line> lines, double characterSpacing, double lineSpacing) {
+
+        double maxHorizontalDistance = 0;
+        double minVerticalDistance = 0;
+        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE;
+
+        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
+
+        lines.forEach(outer -> {
+
+            lines.forEach(inner -> {
+                if (inner != outer) {
+
+                    double horizontalDistance = outer.horizontalDistance(inner);
+                    double verticalDistance = outer.verticalDistance(inner);
+
+                    if (horizontalDistance <= maxHorizontalDistance && minVerticalDistance <= verticalDistance && verticalDistance <= maxVerticalDistance) {
+                        unionFind.union(outer, inner);
+                    } else if (minVerticalDistance <= verticalDistance && verticalDistance <= maxVerticalDistance && Math.abs(horizontalDistance - Math.min(outer.getLength(),
+                            inner.getLength())) < 0.1) {
+                        boolean characterOverlap = false;
+                        int overlappingCount = 0;
+                        for (Character outerCharacter : outer.getCharacters()) {
+                            for (Character innerCharacter : inner.getCharacters()) {
+                                double characterOverlapDistance = outerCharacter.overlappingDistance(innerCharacter);
+                                if (characterOverlapDistance > 2) {
+                                    characterOverlap = true;
+                                }
+                                if (characterOverlapDistance > 0) {
+                                    overlappingCount++;
+                                }
+                            }
+                        }
+                        if (!characterOverlap && overlappingCount <= 2) {
+                            unionFind.union(outer, inner);
+                        }
+                    }
+                }
+            });
+        });
+
+        List<Line> outputZone = new ArrayList<>();
+        for (Set<Line> group : unionFind.getGroups()) {
+            List<Character> characters = new ArrayList<>();
+            for (Line line : group) {
+                characters.addAll(line.getCharacters());
+            }
+            characters.sort(Comparator.comparingDouble(Character::getX));
+
+            outputZone.add(new Line(characters, characterSpacing));
+        }
+
+        return new Zone(outputZone);
+    }
+
 }
@@ -7,12 +7,9 @@ public class DoubleUtils {
        if (Double.isNaN(d1) || Double.isNaN(d2)) {
            return Double.compare(d1, d2);
        }
-        if (precision == 0) {
-            precision = 1;
-        }
-        long i1 = Math.round(d1 / precision);
-        long i2 = Math.round(d2 / precision);
-        return Long.valueOf(i1).compareTo(i2);
+        long i1 = Math.round(d1 / (precision == 0 ? 1 : precision));
+        long i2 = Math.round(d2 / (precision == 0 ? 1 : precision));
+        return Long.compare(i1, i2);
    }

 }
@@ -7,21 +7,24 @@ import static java.util.stream.Collectors.toList;
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
+import java.util.stream.Collectors;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Header;
@@ -31,6 +34,8 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

@@ -44,14 +49,14 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class DocumentGraphFactory {

-    public Document buildDocumentGraph(ClassificationDocument document) {
+    public Document buildDocumentGraph(LayoutParsingType layoutParsingType, ClassificationDocument document) {

        Document documentGraph = new Document();
        Context context = new Context(documentGraph);

        document.getPages().forEach(context::buildAndAddPageWithCounter);
        document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.getImages().add(image));
-        addSections(document, context);
+        addSections(layoutParsingType, document, context);
        addHeaderAndFooterToEachPage(document, context);

        documentGraph.setNumberOfPages(context.pages.size());
@@ -62,9 +67,9 @@ public class DocumentGraphFactory {
    }


-    private void addSections(ClassificationDocument document, Context context) {
+    private void addSections(LayoutParsingType layoutParsingType, ClassificationDocument document, Context context) {

-        document.getSections().forEach(section -> SectionNodeFactory.addSection(null, section.getNonEmptyPageBlocks(), section.getImages(), context));
+        document.getSections().forEach(section -> SectionNodeFactory.addSection(layoutParsingType, null, section.getNonEmptyPageBlocks(), section.getImages(), context));
    }


@@ -75,6 +80,8 @@ public class DocumentGraphFactory {
        GenericSemanticNode node;
        if (originalTextBlock.isHeadline()) {
            node = Headline.builder().documentTree(context.getDocumentTree()).build();
+        } else if (originalTextBlock.isToDuplicate()) {
+            node = DuplicatedParagraph.builder().documentTree(context.getDocumentTree()).build();
        } else {
            node = Paragraph.builder().documentTree(context.getDocumentTree()).build();
        }
@@ -84,7 +91,16 @@ public class DocumentGraphFactory {
        List<TextPageBlock> textBlocks = new ArrayList<>();
        textBlocks.add(originalTextBlock);
        textBlocks.addAll(textBlocksToMerge);
+
        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
+
+        if (node instanceof DuplicatedParagraph duplicatedParagraph) {
+            AtomicTextBlock unsortedTextBlock = context.textBlockFactory.buildAtomicTextBlock(textBlocks.stream()
+                    .flatMap(tb -> tb.getSequences().stream())
+                    .collect(Collectors.toList()), node, context, page);
+            duplicatedParagraph.setUnsortedLeafTextBlock(unsortedTextBlock);
+        }
+
        List<Integer> treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node);
        node.setLeafTextBlock(textBlock);
        node.setTreeId(treeId);
@@ -95,18 +111,21 @@ public class DocumentGraphFactory {

        Rectangle2D position = image.getPosition();
        Page page = context.getPage(image.getPage());
-        Image imageNode = Image.builder()
+        var imageBuilder = Image.builder()
                .id(IdBuilder.buildId(Set.of(page), List.of(position)))
                .imageType(image.getImageType())
                .position(position)
                .transparent(image.isHasTransparency())
                .page(page)
-                .documentTree(context.getDocumentTree())
-                .build();
+                .documentTree(context.getDocumentTree());
+        if (image.isSourceByAi()) {
+            imageBuilder.engines(new HashSet<>(Set.of(LayoutEngine.AI)));
+        }
+        Image imageNode = imageBuilder.build();
        page.getMainBody().add(imageNode);

-        List<Integer> tocId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode);
-        imageNode.setTreeId(tocId);
+        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode);
+        imageNode.setTreeId(treeId);
    }


@@ -4,19 +4,21 @@ import static java.lang.String.format;
 import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.groupingBy;

+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TableMergingUtility;

 import lombok.experimental.UtilityClass;
@@ -24,7 +26,11 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class SectionNodeFactory {

-    public void addSection(GenericSemanticNode parentNode, List<AbstractPageBlock> pageBlocks, List<ClassifiedImage> images, DocumentGraphFactory.Context context) {
+    public void addSection(LayoutParsingType layoutParsingType,
+                           GenericSemanticNode parentNode,
+                           List<AbstractPageBlock> pageBlocks,
+                           List<ClassifiedImage> images,
+                           DocumentGraphFactory.Context context) {

        if (pageBlocks.isEmpty()) {
            return;
@@ -37,11 +43,11 @@ public class SectionNodeFactory {

        section.setTreeId(getTreeId(parentNode, context, section));

-        addFirstHeadlineDirectlyToSection(pageBlocks, context, section);
+        addFirstHeadlineDirectlyToSection(layoutParsingType, pageBlocks, context, section);
        if (containsTablesAndTextBlocks(pageBlocks)) {
-            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(section, subSectionPageBlocks, emptyList(), context));
+            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(layoutParsingType, section, subSectionPageBlocks, emptyList(), context));
        } else {
-            addTablesAndParagraphsAndHeadlinesToSection(pageBlocks, context, section);
+            addTablesAndParagraphsAndHeadlinesToSection(layoutParsingType, pageBlocks, context, section);
        }

        images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(section, image, context));
@@ -58,16 +64,19 @@ public class SectionNodeFactory {
    }


-    private void addFirstHeadlineDirectlyToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
+    private void addFirstHeadlineDirectlyToSection(LayoutParsingType layoutParsingType, List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {

        if (pageBlocks.get(0).isHeadline()) {
-            addTablesAndParagraphsAndHeadlinesToSection(List.of(pageBlocks.get(0)), context, section);
+            addTablesAndParagraphsAndHeadlinesToSection(layoutParsingType, List.of(pageBlocks.get(0)), context, section);
            pageBlocks.remove(0);
        }
    }


-    private void addTablesAndParagraphsAndHeadlinesToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
+    private void addTablesAndParagraphsAndHeadlinesToSection(LayoutParsingType layoutParsingType,
+                                                             List<AbstractPageBlock> pageBlocks,
+                                                             DocumentGraphFactory.Context context,
+                                                             Section section) {

        Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
        List<AbstractPageBlock> remainingBlocks = new LinkedList<>(pageBlocks);
@@ -80,13 +89,23 @@ public class SectionNodeFactory {
            remainingBlocks.removeAll(alreadyMerged);

            if (abstractPageBlock instanceof TextPageBlock) {
-                List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY((TextPageBlock) abstractPageBlock, remainingBlocks);
-                alreadyMerged.addAll(textBlocks);
-                DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
+
+                switch (layoutParsingType) {
+                    case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> {
+                        alreadyMerged.add(abstractPageBlock);
+                        remainingBlocks.remove(abstractPageBlock);
+                        DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, new ArrayList<>());
+                    }
+                    default -> {
+                        List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY((TextPageBlock) abstractPageBlock, remainingBlocks);
+                        alreadyMerged.addAll(textBlocks);
+                        DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
+                    }
+                }
            } else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
                List<TablePageBlock> tablesToMerge = TableMergingUtility.findConsecutiveTablesWithSameColCountAndSameHeaders(tablePageBlock, remainingBlocks);
                alreadyMerged.addAll(tablesToMerge);
-                TableNodeFactory.addTable(section, tablesToMerge, context);
+                TableNodeFactory.addTable(layoutParsingType, section, tablesToMerge, context);
            } else {
                throw new RuntimeException(format("Unhandled AbstractPageBlockType %s!", abstractPageBlock.getClass()));
            }
@@ -171,6 +190,7 @@ public class SectionNodeFactory {
                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
                .map(abstractTextContainer -> (TextPageBlock) abstractTextContainer)
                .filter(abstractTextContainer -> abstractTextContainer.getDir() == atc.getDir())
+                .filter(abstractTextContainer -> !abstractTextContainer.isToDuplicate())
                .toList();
    }

@@ -7,16 +7,17 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;

 import lombok.experimental.UtilityClass;
@@ -27,7 +28,7 @@ public class TableNodeFactory {
    public final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;


-    public void addTable(GenericSemanticNode parentNode, List<TablePageBlock> tablesToMerge, DocumentGraphFactory.Context context) {
+    public void addTable(LayoutParsingType layoutParsingType, GenericSemanticNode parentNode, List<TablePageBlock> tablesToMerge, DocumentGraphFactory.Context context) {

        setPageNumberInCells(tablesToMerge);
        Set<Page> pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet());
@@ -43,7 +44,7 @@ public class TableNodeFactory {

        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table);
        table.setTreeId(treeId);
-        addTableCells(mergedRows, table, context);
+        addTableCells(layoutParsingType, mergedRows, table, context);

        ifTableHasNoHeadersSetFirstRowAsHeaders(table);
    }
@@ -88,18 +89,18 @@ public class TableNodeFactory {
    }


-    private void addTableCells(List<List<Cell>> rows, Table table, DocumentGraphFactory.Context context) {
+    private void addTableCells(LayoutParsingType layoutParsingType, List<List<Cell>> rows, Table table, DocumentGraphFactory.Context context) {

        for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
            for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, table, context);
+                addTableCell(layoutParsingType, rows.get(rowIndex).get(colIndex), rowIndex, colIndex, table, context);
            }
        }
    }


    @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong
-    private void addTableCell(Cell cell, int rowIndex, int colIndex, Table tableNode, DocumentGraphFactory.Context context) {
+    private void addTableCell(LayoutParsingType layoutParsingType, Cell cell, int rowIndex, int colIndex, Table tableNode, DocumentGraphFactory.Context context) {

        Page page = context.getPage(cell.getPageNumber());

@@ -116,7 +117,7 @@ public class TableNodeFactory {
            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page);
            tableCell.setLeafTextBlock(textBlock);
        } else if (firstTextBlockIsHeadline(cell)) {
-            SectionNodeFactory.addSection(tableCell, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
+            SectionNodeFactory.addSection(layoutParsingType, tableCell, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
        } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
            List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
            textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -12,8 +13,10 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
@@ -73,17 +76,24 @@ public class DocumentDataMapper {
            case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode());
            case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode());
            case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode());
+            case PARAGRAPH ->
+                    entry.getNode() instanceof DuplicatedParagraph duplicatedParagraph ? PropertiesMapper.buildDuplicateParagraphProperties(duplicatedParagraph) : new HashMap<>();
            default -> new HashMap<>();
        };

-        return DocumentStructure.EntryData.builder()
+        DocumentStructure.EntryData.EntryDataBuilder documentBuilder = DocumentStructure.EntryData.builder()
                .treeId(toPrimitiveIntArray(entry.getTreeId()))
                .children(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
                .type(entry.getType())
                .atomicBlockIds(atomicTextBlocks)
                .pageNumbers(entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new))
-                .properties(properties)
-                .build();
+                .properties(properties);
+        if (entry.getNode() != null) {
+            documentBuilder.engines(entry.getNode().getEngines());
+        } else {
+            documentBuilder.engines(new HashSet<>(Set.of(LayoutEngine.ALGORITHM)));
+        }
+        return documentBuilder.build();
    }


@@ -7,13 +7,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Header;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
@@ -61,7 +62,7 @@ public class DocumentGraphMapper {

            SemanticNode node = switch (entryData.getType()) {
                case SECTION -> buildSection(context);
-                case PARAGRAPH -> buildParagraph(context);
+                case PARAGRAPH -> buildParagraph(context, entryData.getProperties());
                case HEADLINE -> buildHeadline(context);
                case HEADER -> buildHeader(context);
                case FOOTER -> buildFooter(context);
@@ -140,7 +141,17 @@ public class DocumentGraphMapper {
    }


-    private Paragraph buildParagraph(Context context) {
+    private Paragraph buildParagraph(Context context, Map<String, String> properties) {
+
+        if (PropertiesMapper.isDuplicateParagraph(properties)) {
+
+            DuplicatedParagraph duplicatedParagraph = DuplicatedParagraph.builder().documentTree(context.documentTree).build();
+
+            Long[] unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
+            duplicatedParagraph.setUnsortedLeafTextBlock(toTextBlock(unsortedTextblockIds, context, duplicatedParagraph));
+            return duplicatedParagraph;
+
+        }

        return Paragraph.builder().documentTree(context.documentTree).build();
    }
@@ -1,17 +1,19 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;

 import java.awt.geom.Rectangle2D;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

 public class PropertiesMapper {

@@ -76,6 +78,32 @@ public class PropertiesMapper {
    }


+    public static Map<String, String> buildDuplicateParagraphProperties(DuplicatedParagraph duplicatedParagraph) {
+
+        Map<String, String> properties = new HashMap<>();
+        properties.put(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID, Arrays.toString(toAtomicTextBlockIds(duplicatedParagraph.getUnsortedLeafTextBlock())));
+        return properties;
+    }
+
+
+    public static boolean isDuplicateParagraph(Map<String, String> properties) {
+
+        return properties.containsKey(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
+    }
+
+
+    public static Long[] getUnsortedTextblockIds(Map<String, String> properties) {
+
+        return toLongArray(properties.get(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
+    }
+
+
+    public static Long[] toLongArray(String ids) {
+
+        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toArray(Long[]::new);
+    }
+
+
    private static ImageType parseImageType(String imageType) {

        return switch (imageType) {
@@ -101,4 +129,10 @@ public class PropertiesMapper {
                rectangle2D.getHeight());
    }

+
+    private static Long[] toAtomicTextBlockIds(TextBlock textBlock) {
+
+        return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);
+    }
+
 }
@@ -237,8 +237,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        int startIndex = 0;
        RedTextPosition previous = null;

+        float direction = -1;
        for (int i = 0; i <= textPositions.size() - 1; i++) {

+            if (direction == -1) {
+                direction = textPositions.get(i).getDir();
+            }
+
            if (!textPositionSequences.isEmpty()) {
                previous = textPositionSequences.get(textPositionSequences.size() - 1)
                        .getTextPositions()
@@ -250,6 +255,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                continue;
            }

+            if (textPositions.get(i).getDir() != direction && startIndex != i) {
+                List<TextPosition> sublist = textPositions.subList(startIndex, i);
+                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
+                startIndex = i;
+                direction = textPositions.get(i).getDir();
+            }
+
            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
@@ -329,6 +341,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
    }

+
    @Override
    public String getText(PDDocument doc) throws IOException {

@@ -17,8 +17,10 @@ import java.util.stream.Stream;

 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
@@ -52,6 +54,8 @@ public class LayoutGridService {

    static Color INNER_LINES_COLOR = new Color(255, 175, 175);
    static Color PARAGRAPH_COLOR = new Color(70, 130, 180);
+
+    static Color DUPLICATE_PARAGRAPH_COLOR = new Color(70, 180, 101);
    static Color TABLE_COLOR = new Color(102, 205, 170);
    static Color SECTION_COLOR = new Color(50, 50, 50);
    static Color HEADLINE_COLOR = new Color(162, 56, 56);
@@ -63,43 +67,58 @@ public class LayoutGridService {
    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
    public void addLayoutGrid(File originFile, Document document, File destinationFile, boolean layerVisibilityDefaultValue) {

-        LayoutGrid layoutGrid = createLayoutGrid(document);
+        Visualizations layoutGrid = this.addLayoutGrid(document, layerVisibilityDefaultValue, false);
+        Visualizations visualLayoutGrid = this.addLayoutGrid(document, layerVisibilityDefaultValue, true);

-        viewerDocumentService.addVisualizationsOnPage(originFile,
-                destinationFile,
-                Visualizations.builder()
-                        .layer(ContentStreams.KNECON_LAYOUT)
-                        .visualizationsOnPages(layoutGrid.getVisualizationsPerPages())
-                        .layerVisibilityDefaultValue(layerVisibilityDefaultValue)
-                        .build());
+        viewerDocumentService.addVisualizationsOnPage(originFile, destinationFile, List.of(layoutGrid, visualLayoutGrid));
    }


-    private LayoutGrid createLayoutGrid(Document document) {
+    @SneakyThrows
+    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
+    public Visualizations addLayoutGrid(Document document, boolean layerVisibilityDefaultValue, boolean visualParsingGrid) {
+
+        LayoutGrid layoutGrid = createLayoutGrid(document, visualParsingGrid);
+
+        return Visualizations.builder()
+                .layer(visualParsingGrid ? ContentStreams.KNECON_VISUAL_PARSING : ContentStreams.KNECON_LAYOUT)
+                .visualizationsOnPages(layoutGrid.getVisualizationsPerPages())
+                .layerVisibilityDefaultValue(layerVisibilityDefaultValue)
+                .build();
+    }
+
+
+    private LayoutGrid createLayoutGrid(Document document, boolean visualParsingGrid) {

        LayoutGrid layoutGrid = new LayoutGrid(document.getNumberOfPages());
-        document.streamAllSubNodes().forEach(semanticNode -> {
-            Color color = switch (semanticNode.getType()) {
-                case PARAGRAPH -> PARAGRAPH_COLOR;
-                case TABLE -> TABLE_COLOR;
-                case SECTION -> SECTION_COLOR;
-                case HEADLINE -> HEADLINE_COLOR;
-                case HEADER, FOOTER -> HEADER_COLOR;
-                case IMAGE -> IMAGE_COLOR;
-                default -> null;
-            };
-            if (isNotSectionOrTableCellOrDocument(semanticNode)) {
-                addAsRectangle(semanticNode, layoutGrid, color);
-            }
-            if (semanticNode.getType().equals(NodeType.SECTION)) {
-                addSection(semanticNode, layoutGrid, color);
-            }
-            if (semanticNode.getType().equals(NodeType.TABLE)) {
-                Table table = (Table) semanticNode;
-                addInnerTableLines(table, layoutGrid);
-            }
-        });
+        document.streamAllSubNodes()
+                .filter(node -> (node.getEngines().contains(LayoutEngine.AI) && visualParsingGrid) || (node.getEngines().contains(LayoutEngine.ALGORITHM) && !visualParsingGrid))
+                .forEach(semanticNode -> {
+                    Color color = switch (semanticNode.getType()) {
+                        case PARAGRAPH -> PARAGRAPH_COLOR;
+                        case TABLE -> TABLE_COLOR;
+                        case SECTION -> SECTION_COLOR;
+                        case HEADLINE -> HEADLINE_COLOR;
+                        case HEADER, FOOTER -> HEADER_COLOR;
+                        case IMAGE -> IMAGE_COLOR;
+                        default -> null;
+                    };

+                    if (semanticNode instanceof DuplicatedParagraph) {
+                        color = DUPLICATE_PARAGRAPH_COLOR;
+                    }
+
+                    if (isNotSectionOrTableCellOrDocument(semanticNode)) {
+                        addAsRectangle(semanticNode, layoutGrid, color);
+                    }
+                    if (semanticNode.getType().equals(NodeType.SECTION)) {
+                        addSection(semanticNode, layoutGrid, color);
+                    }
+                    if (semanticNode.getType().equals(NodeType.TABLE)) {
+                        Table table = (Table) semanticNode;
+                        addInnerTableLines(table, layoutGrid);
+                    }
+                });
        return layoutGrid;
    }

@@ -188,14 +207,33 @@ public class LayoutGridService {
    @SneakyThrows
    private void addPlacedText(Page page, Rectangle2D textBBox, String s, LayoutGrid layoutGrid) {

-        Point2D.Float upperLeftCorner = switch (page.getRotation()) {
-            case 90 -> new Point2D.Float((float) (textBBox.getMinX()), (float) textBBox.getMinY());
-            case 180 -> new Point2D.Float((float) (textBBox.getMaxX()), (float) textBBox.getMinY());
-            case 270 -> new Point2D.Float((float) (textBBox.getMaxX()), (float) textBBox.getMaxY());
-            default -> new Point2D.Float((float) (textBBox.getMinX()), (float) textBBox.getMaxY());
-        };
+        // translates text, such that its right edge is a bit to the left of the drawn box
+        float translationAmount = ((FONT.getStringWidth(s) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4);
+
+        Point2D upperLeftCorner;
+        Point2D translationVector;
+        switch (page.getRotation()) {
+            case 90 -> {
+                upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMinY());
+                translationVector = new Point2D.Double(FONT_SIZE, -translationAmount);
+            }
+            case 180 -> {
+                upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMinY());
+                translationVector = new Point2D.Double(translationAmount, FONT_SIZE);
+            }
+            case 270 -> {
+                upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMaxY());
+                translationVector = new Point2D.Double(-FONT_SIZE, translationAmount);
+            }
+            default -> {
+                upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMaxY());
+                translationVector = new Point2D.Double(-translationAmount, -FONT_SIZE);
+            }
+        }
+
+        upperLeftCorner = add(upperLeftCorner, translationVector);
+
        var placedTexts = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getPlacedTexts();
-        upperLeftCorner.setLocation(upperLeftCorner.getX() - ((FONT.getStringWidth(s) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), upperLeftCorner.getY() - FONT_SIZE);
        placedTexts.add(PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, Color.BLACK, FONT));
    }

@@ -317,4 +355,10 @@ public class LayoutGridService {
                        .add(new ColoredRectangle(textBBox, color, LINE_WIDTH)));
    }

+
+    private Point2D add(Point2D a, Point2D b) {
+
+        return new Point2D.Double(a.getX() + b.getX(), a.getY() + b.getY());
+    }
+
 }
@@ -1,270 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.services.visualization;
-
-import java.awt.Color;
-import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.PDPageContentStream;
-import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
-import org.apache.pdfbox.util.Matrix;
-import org.springframework.core.io.ClassPathResource;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
-import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class PdfDraw {
-
-    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) throws IOException {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
-                        pageNumber,
-                        rectanglesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-
-    }
-
-
-    public static void drawRectanglesPerPageNumberedByLine(String filename, List<List<List<Rectangle2D>>> rectanglesPerPage, String tmpFileName) throws IOException {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                var rectanglesOnPage = rectanglesPerPage.get(pageNumber - 1);
-                for (int lineNumber = 0; lineNumber < rectanglesOnPage.size(); lineNumber++) {
-                    var rectanglesInLine = rectanglesOnPage.get(lineNumber);
-                    PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, rectanglesInLine, PdfVisualisationUtility.Options.builder().stroke(true).build());
-                    double y = Math.min(rectanglesInLine.get(0).getMinY(), rectanglesInLine.get(0).getMaxY());
-                    PdfVisualisationUtility.drawText(String.format("%d", lineNumber),
-                            pdDocument,
-                            new Point2D.Double(rectanglesInLine.get(0).getX() - (5 + (5 * countNumberOfDigits(lineNumber))), y + 2),
-                            pageNumber,
-                            PdfVisualisationUtility.Options.builder().stroke(true).build());
-                }
-            }
-            pdDocument.save(out);
-        }
-
-    }
-
-
-    private static int countNumberOfDigits(int num) {
-
-        int final_num = num;
-        if (final_num == 0) {
-            return 1;
-        }
-        int count = 0;
-        for (; final_num != 0; final_num /= 10) {
-            count++;
-        }
-        return count;
-    }
-
-
-    public static void drawDocumentGraph(PDDocument document, Document documentGraph) {
-
-        documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry));
-    }
-
-
-    public static void drawNode(PDDocument document, DocumentTree.Entry entry) {
-
-        Options options = buildStandardOptionsForNodes(entry);
-
-        drawBBoxAndLabelAndNumberOnPage(document, entry, options);
-
-    }
-
-
-    public static void drawTextBlock(PDDocument document, TextBlock textBlock, Options options) {
-
-        textBlock.getAtomicTextBlocks().forEach(atb -> drawAtomicTextBlock(document, atb, options));
-    }
-
-
-    public static void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, Options options) {
-
-        drawRectangle2DList(document, atomicTextBlock.getPage().getNumber(), atomicTextBlock.getPositions().stream().toList(), options);
-
-    }
-
-
-    @SneakyThrows
-    private static void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, Options options, boolean rotate) {
-
-        var pdPage = document.getPage(pageNumber - 1);
-        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
-
-        contentStream.setNonStrokingColor(options.getStrokeColor());
-        contentStream.setLineWidth(options.getStrokeWidth());
-
-        contentStream.beginText();
-        if (rotate) {
-            contentStream.setTextMatrix(Matrix.getRotateInstance(Math.toRadians(15), (float) location.getX(), (float) location.getY()));
-        } else {
-            contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
-        }
-        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
-        contentStream.showText(string);
-        contentStream.endText();
-        contentStream.close();
-    }
-
-
-    @SneakyThrows
-    public static void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, Options options) {
-
-        var pdPage = document.getPage(pageNumber - 1);
-        drawRectangle2DList(document, rectCollection, options, pdPage);
-    }
-
-
-    private static void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, Options options, PDPage pdPage) throws IOException {
-
-        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
-
-        contentStream.setStrokingColor(options.getStrokeColor());
-        contentStream.setNonStrokingColor(options.getFillColor());
-        contentStream.setLineWidth(options.getStrokeWidth());
-
-        for (var r : rectCollection) {
-            contentStream.addRect((float) r.getMinX(), (float) r.getMinY(), (float) r.getWidth(), (float) r.getHeight());
-
-            if (options.isStroke() && options.isFill()) {
-                contentStream.fillAndStroke();
-            } else if (options.isStroke()) {
-                contentStream.stroke();
-            } else if (options.isFill()) {
-                contentStream.fill();
-            }
-        }
-        contentStream.close();
-    }
-
-
-    @SneakyThrows
-    public static void drawRectanglesAndLinesPerPage(String filename, List<List<Rectangle2D>> list, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-//                PdfVisualisationUtility.drawLine2DList(pdDocument,
-//                        pageNumber,
-//                        list.get(pageNumber - 1),
-//                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
-                        pageNumber,
-                        rectanglesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-    }
-
-
-    @SneakyThrows
-    public static void drawLinesPerPage(String filename, List<List<Ruling>> linesPerPage, String tmpFileName) {
-
-        ClassPathResource pdfResource = new ClassPathResource(filename);
-        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
-            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
-                PdfVisualisationUtility.drawLine2DList(pdDocument,
-                        pageNumber,
-                        linesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
-            }
-            pdDocument.save(out);
-        }
-    }
-
-
-    @Builder
-    @AllArgsConstructor
-    @Getter
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    public static class Options {
-
-        boolean stroke;
-        @Builder.Default
-        Color strokeColor = Color.BLACK;
-        @Builder.Default
-        float strokeWidth = 1f;
-
-        boolean fill;
-        @Builder.Default
-        Color fillColor = Color.BLACK;
-
-    }
-
-
-    private static Options buildStandardOptionsForNodes(DocumentTree.Entry entry) {
-
-        return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
-            case DOCUMENT -> Color.LIGHT_GRAY;
-            case HEADER, FOOTER -> Color.GREEN;
-            case PARAGRAPH -> Color.BLUE;
-            case HEADLINE -> Color.RED;
-            case SECTION -> Color.BLACK;
-            case TABLE -> Color.ORANGE;
-            case TABLE_CELL -> Color.GRAY;
-            case IMAGE -> Color.MAGENTA;
-        }).build();
-    }
-
-
-    private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, Options options) {
-
-        Map<Page, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
-        for (Page page : rectanglesPerPage.keySet()) {
-            Rectangle2D rectangle2D = rectanglesPerPage.get(page);
-            if (entry.getType() == NodeType.SECTION) {
-                rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
-            }
-            drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options);
-            drawText(buildString(entry),
-                    document,
-                    new Point2D.Double(rectangle2D.getMinX(), rectangle2D.getMaxY() + 2),
-                    page.getNumber(),
-                    options,
-                    entry.getType() == NodeType.TABLE_CELL);
-        }
-    }
-
-
-    private static String buildString(DocumentTree.Entry entry) {
-
-        return entry.getNode().getNumberOnPage() + ": " + entry.getTreeId() + ": " + entry.getType();
-    }
-
-}
@@ -0,0 +1,28 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import java.awt.Color;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Builder
+@AllArgsConstructor
+@Getter
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DrawingOptions {
+
+    boolean stroke;
+    @Builder.Default
+    Color strokeColor = Color.BLACK;
+    @Builder.Default
+    float strokeWidth = 1f;
+
+    boolean fill;
+    @Builder.Default
+    Color fillColor = Color.BLACK;
+
+}
+
@@ -0,0 +1,88 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import java.awt.geom.Point2D;
+import java.util.Comparator;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Rectangle;
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+
+public class GeometricComparators {
+
+    private static final int COMPARATOR_ROUNDING = 2;
+
+    public static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (point1, point2) -> {
+
+        int rv = 0;
+        float point1X = DoubleComparisons.round(point1.getX(), COMPARATOR_ROUNDING);
+        float point1Y = DoubleComparisons.round(point1.getY(), COMPARATOR_ROUNDING);
+        float point2X = DoubleComparisons.round(point2.getX(), COMPARATOR_ROUNDING);
+        float point2Y = DoubleComparisons.round(point2.getY(), COMPARATOR_ROUNDING);
+
+        if (point1X > point2X) {
+            rv = 1;
+        } else if (point1X < point2X) {
+            rv = -1;
+        } else if (point1Y > point2Y) {
+            rv = 1;
+        } else if (point1Y < point2Y) {
+            rv = -1;
+        }
+        return rv;
+    };
+
+    public static final Comparator<Point2D> Y_FIRST_POINT_COMPARATOR = (point1, point2) -> {
+
+        int rv = 0;
+        float point1X = DoubleComparisons.round(point1.getX(), COMPARATOR_ROUNDING);
+        float point1Y = DoubleComparisons.round(point1.getY(), COMPARATOR_ROUNDING);
+        float point2X = DoubleComparisons.round(point2.getX(), COMPARATOR_ROUNDING);
+        float point2Y = DoubleComparisons.round(point2.getY(), COMPARATOR_ROUNDING);
+
+        if (point1Y > point2Y) {
+            rv = 1;
+        } else if (point1Y < point2Y) {
+            rv = -1;
+        } else if (point1X > point2X) {
+            rv = 1;
+        } else if (point1X < point2X) {
+            rv = -1;
+        }
+        return rv;
+    };
+
+    public static final Comparator<Cell> CELL_SIZE_COMPARATOR = (cell1, cell2) -> {
+
+        Double cell1Size = cell1.getHeight() * cell1.getWidth();
+        Double cell2Size = cell2.getHeight() * cell2.getWidth();
+        return cell1Size.compareTo(cell2Size);
+    };
+
+    public static final Comparator<Rectangle> RECTANGLE_SIZE_COMPARATOR = (rect1, rect2) -> {
+
+        Double rect1Size = rect1.getHeight() * rect1.getWidth();
+        Double rect2Size = rect2.getHeight() * rect2.getWidth();
+        return rect1Size.compareTo(rect2Size);
+    };
+
+    public static final Comparator<Ruling> X_FIRST_RULING_COMPARATOR = (ruling1, ruling2) -> {
+
+        int rv = 0;
+        float point1X = DoubleComparisons.round(Math.min(ruling1.getLeft(), ruling1.getRight()), COMPARATOR_ROUNDING);
+        float point1Y = DoubleComparisons.round(Math.min(ruling1.getTop(), ruling1.getBottom()), COMPARATOR_ROUNDING);
+        float point2X = DoubleComparisons.round(Math.min(ruling2.getLeft(), ruling2.getRight()), COMPARATOR_ROUNDING);
+        float point2Y = DoubleComparisons.round(Math.min(ruling2.getTop(), ruling2.getBottom()), COMPARATOR_ROUNDING);
+
+        if (point1X > point2X) {
+            rv = 1;
+        } else if (point1X < point2X) {
+            rv = -1;
+        } else if (point1Y > point2Y) {
+            rv = 1;
+        } else if (point1Y < point2Y) {
+            rv = -1;
+        }
+        return rv;
+    };
+
+}
@@ -21,11 +21,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

-import lombok.AccessLevel;
-import lombok.Builder;
-import lombok.Getter;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

@@ -41,20 +37,20 @@ public class PdfVisualisationUtility {

    public void drawNode(PDDocument document, DocumentTree.Entry entry) {

-        Options options = buildStandardOptionsForNodes(entry);
+        DrawingOptions options = buildStandardOptionsForNodes(entry);

        drawBBoxAndLabelAndNumberOnPage(document, entry, options);

    }


-    public void drawTextBlock(PDDocument document, TextBlock textBlock, Options options) {
+    public void drawTextBlock(PDDocument document, TextBlock textBlock, DrawingOptions options) {

        textBlock.getAtomicTextBlocks().forEach(atb -> drawAtomicTextBlock(document, atb, options));
    }


-    public void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, Options options) {
+    public void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, DrawingOptions options) {

        drawRectangle2DList(document, atomicTextBlock.getPage().getNumber(), atomicTextBlock.getPositions().stream().toList(), options);

@@ -62,7 +58,7 @@ public class PdfVisualisationUtility {


    @SneakyThrows
-    public void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, Options options) {
+    public void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, DrawingOptions options) {

        var pdPage = document.getPage(pageNumber - 1);
        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
@@ -80,14 +76,14 @@ public class PdfVisualisationUtility {


    @SneakyThrows
-    public void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, Options options) {
+    public void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, DrawingOptions options) {

        var pdPage = document.getPage(pageNumber - 1);
        drawRectangle2DList(document, rectCollection, options, pdPage);
    }


-    private void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, Options options, PDPage pdPage) throws IOException {
+    private void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, DrawingOptions options, PDPage pdPage) throws IOException {

        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);

@@ -110,9 +106,9 @@ public class PdfVisualisationUtility {
    }


-    private Options buildStandardOptionsForNodes(DocumentTree.Entry entry) {
+    private DrawingOptions buildStandardOptionsForNodes(DocumentTree.Entry entry) {

-        return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
+        return DrawingOptions.builder().stroke(true).strokeColor(switch (entry.getType()) {
            case DOCUMENT -> Color.LIGHT_GRAY;
            case HEADER, FOOTER -> Color.GREEN;
            case PARAGRAPH -> Color.BLUE;
@@ -125,7 +121,7 @@ public class PdfVisualisationUtility {
    }


-    private void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, Options options) {
+    private void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, DrawingOptions options) {

        Map<Page, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
        rectanglesPerPage.forEach((page, rectangle2D) -> {
@@ -152,7 +148,7 @@ public class PdfVisualisationUtility {


    @SneakyThrows
-    public static void drawLine2DList(PDDocument pdDocument, int pageNumber, List<? extends Line2D> line2DS, Options options) {
+    public static void drawLine2DList(PDDocument pdDocument, int pageNumber, List<? extends Line2D> line2DS, DrawingOptions options) {

        var pdPage = pdDocument.getPage(pageNumber - 1);
        var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true);
@@ -176,21 +172,4 @@ public class PdfVisualisationUtility {
        contentStream.close();
    }

-
-    @Builder
-    @Getter
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    public static class Options {
-
-        boolean fill;
-        boolean stroke;
-        @Builder.Default
-        Color strokeColor = Color.BLACK;
-        @Builder.Default
-        float strokeWidth = 1f;
-        @Builder.Default
-        Color fillColor = Color.BLACK;
-
-    }
-
 }
@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;

 import static java.lang.String.format;

+import java.awt.geom.Area;
 import java.awt.geom.Rectangle2D;
 import java.awt.geom.RectangularShape;
 import java.util.Collections;
@@ -37,15 +38,28 @@ public class RectangleTransformations {
    }


+    public static double calculateIntersectedArea(Rectangle2D r1, Rectangle2D r2) {
+
+        Area a1 = new Area(r1);
+        Area a2 = new Area(r2);
+        a1.intersect(a2);
+        Rectangle2D intersection = a1.getBounds2D();
+        return intersection.getWidth() * intersection.getHeight();
+    }
+
+
    public static Rectangle2D bBoxUnionAtomicTextBlock(List<AtomicTextBlock> atomicTextBlocks) {

        return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DBBoxCollector());
    }
+
+
    public static Collector<Rectangle2D, Rectangle2DBBoxCollector.BBox, Rectangle2D> collectBBox() {

        return new Rectangle2DBBoxCollector();
    }

+
    public static PDRectangle toPDRectangleBBox(List<Rectangle> rectangles) {

        Rectangle2D rectangle2D = RectangleTransformations.rectangleBBox(rectangles);
@@ -70,6 +84,7 @@ public class RectangleTransformations {
        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
    }

+
    public static Rectangle2D rectangleBBox(List<Rectangle> rectangles) {

        return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DBBoxCollector());
@@ -84,6 +99,7 @@ public class RectangleTransformations {
                -redactionLogRectangle.getHeight());
    }

+
    public static Rectangle2D toRectangle2D(PDRectangle rectangle) {

        return new Rectangle2D.Double(rectangle.getLowerLeftX(), rectangle.getLowerLeftY(), rectangle.getWidth(), rectangle.getHeight());
@@ -0,0 +1,77 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import static com.knecon.fforesight.service.layoutparser.processor.utils.GeometricComparators.Y_FIRST_POINT_COMPARATOR;
+
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+
+public class RectangularIntersectionFinder {
+
+    public static List<Rectangle2D> find(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+
+        // Fix for 211.pdf
+        for (Ruling r : horizontalRulingLines) {
+            if (r.getX2() < r.getX1()) {
+                double a = r.getX2();
+                r.x2 = (float) r.getX1();
+                r.x1 = (float) a;
+            }
+        }
+
+        List<Rectangle2D> foundRectangles = new ArrayList<>();
+        Map<Point2D, Ruling[]> intersectionPoints = Ruling.findIntersections(horizontalRulingLines, verticalRulingLines);
+        List<Point2D> intersectionPointsList = new ArrayList<>(intersectionPoints.keySet());
+        intersectionPointsList.sort(Y_FIRST_POINT_COMPARATOR);
+
+        for (int i = 0; i < intersectionPointsList.size(); i++) {
+            Point2D topLeft = intersectionPointsList.get(i);
+            Ruling[] hv = intersectionPoints.get(topLeft);
+
+            // CrossingPointsDirectlyBelow( topLeft );
+            List<Point2D> xPoints = new ArrayList<>();
+            // CrossingPointsDirectlyToTheRight( topLeft );
+            List<Point2D> yPoints = new ArrayList<>();
+
+            for (Point2D p : intersectionPointsList.subList(i, intersectionPointsList.size())) {
+                if (p.getX() == topLeft.getX() && p.getY() > topLeft.getY()) {
+                    xPoints.add(p);
+                }
+                if (p.getY() == topLeft.getY() && p.getX() > topLeft.getX()) {
+                    yPoints.add(p);
+                }
+            }
+            outer:
+            for (Point2D xPoint : xPoints) {
+                // is there a vertical edge b/w topLeft and xPoint?
+                if (!hv[1].equals(intersectionPoints.get(xPoint)[1])) {
+                    continue;
+                }
+                for (Point2D yPoint : yPoints) {
+                    // is there a horizontal edge b/w topLeft and yPoint ?
+                    if (!hv[0].equals(intersectionPoints.get(yPoint)[0])) {
+                        continue;
+                    }
+                    Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY());
+                    if (intersectionPoints.containsKey(btmRight)
+                        && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0])
+                        && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
+                        foundRectangles.add(new Rectangle2D.Double(topLeft.getX(), topLeft.getY(), btmRight.getX() - topLeft.getX(), btmRight.getY() - topLeft.getY()));
+                        break outer;
+                    }
+                }
+            }
+        }
+
+        // TODO create cells for vertical ruling lines with aligned endpoints at the top/bottom of a grid
+        // that aren't connected with an horizontal ruler?
+        // see: https://github.com/jazzido/tabula-extractor/issues/78#issuecomment-41481207
+
+        return foundRectangles;
+    }
+
+}
@@ -0,0 +1,172 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import static com.knecon.fforesight.service.layoutparser.processor.utils.GeometricComparators.X_FIRST_POINT_COMPARATOR;
+import static com.knecon.fforesight.service.layoutparser.processor.utils.GeometricComparators.Y_FIRST_POINT_COMPARATOR;
+
+import java.awt.geom.Point2D;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Rectangle;
+
+public class SpreadsheetFinder {
+
+    private static final int MAX_OUTER_POINT_TOLERANCE = 10;
+    private static final float AREA_TOLERANCE = 0.001f;
+
+
+    public static List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
+        // via: http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon
+        List<Rectangle> rectangles = new ArrayList<>();
+        Set<Point2D> pointSet = new HashSet<>();
+        Map<Point2D, Point2D> edgesH = new HashMap<>();
+        Map<Point2D, Point2D> edgesV = new HashMap<>();
+
+        for (Rectangle cell : cells) {
+            for (Point2D pt : cell.getPoints()) {
+                if (pointSet.contains(pt)) { // shared vertex, remove it
+                    pointSet.remove(pt);
+                } else {
+                    pointSet.add(pt);
+                }
+            }
+        }
+
+        // X first sort
+        List<Point2D> pointsSortX = new ArrayList<>(pointSet);
+        pointsSortX.sort(X_FIRST_POINT_COMPARATOR);
+        // Y first sort
+        List<Point2D> pointsSortY = new ArrayList<>(pointSet);
+        pointsSortY.sort(Y_FIRST_POINT_COMPARATOR);
+
+        int i = 0;
+        while (i < pointSet.size()) {
+            float currY = (float) pointsSortY.get(i).getY();
+            while (i < pointSet.size() && DoubleComparisons.feq(pointsSortY.get(i).getY(), currY)) {
+                edgesH.put(pointsSortY.get(i), pointsSortY.get(i + 1));
+                edgesH.put(pointsSortY.get(i + 1), pointsSortY.get(i));
+                i += 2;
+            }
+        }
+
+        i = 0;
+        while (i < pointSet.size()) {
+            float currX = (float) pointsSortX.get(i).getX();
+            while (i < pointSet.size() && DoubleComparisons.feq(pointsSortX.get(i).getX(), currX)) {
+                edgesV.put(pointsSortX.get(i), pointsSortX.get(i + 1));
+                edgesV.put(pointsSortX.get(i + 1), pointsSortX.get(i));
+                i += 2;
+            }
+        }
+
+        // Get all the polygons
+        List<List<PolygonVertex>> polygons = new ArrayList<>();
+        Point2D nextVertex;
+        while (!edgesH.isEmpty()) {
+            ArrayList<PolygonVertex> polygon = new ArrayList<>();
+            Point2D first = edgesH.keySet()
+                    .iterator().next();
+            polygon.add(new PolygonVertex(first, Direction.HORIZONTAL));
+            edgesH.remove(first);
+
+            while (true) {
+                PolygonVertex curr = polygon.get(polygon.size() - 1);
+                PolygonVertex lastAddedVertex;
+                if (curr.direction == Direction.HORIZONTAL) {
+                    nextVertex = edgesV.get(curr.point);
+                    edgesV.remove(curr.point);
+                    lastAddedVertex = new PolygonVertex(nextVertex, Direction.VERTICAL);
+                } else {
+                    nextVertex = edgesH.get(curr.point);
+                    edgesH.remove(curr.point);
+                    lastAddedVertex = new PolygonVertex(nextVertex, Direction.HORIZONTAL);
+                }
+                polygon.add(lastAddedVertex);
+
+                if (lastAddedVertex.equals(polygon.get(0))) {
+                    // closed polygon
+                    polygon.remove(polygon.size() - 1);
+                    break;
+                }
+            }
+
+            for (PolygonVertex vertex : polygon) {
+                edgesH.remove(vertex.point);
+                edgesV.remove(vertex.point);
+            }
+            polygons.add(polygon);
+        }
+
+        // calculate grid-aligned minimum area rectangles for each found polygon
+        for (List<PolygonVertex> poly : polygons) {
+            float top = Float.MAX_VALUE;
+            float left = Float.MAX_VALUE;
+            float bottom = Float.MIN_VALUE;
+            float right = Float.MIN_VALUE;
+            for (PolygonVertex pt : poly) {
+                top = (float) Math.min(top, pt.point.getY());
+                left = (float) Math.min(left, pt.point.getX());
+                bottom = (float) Math.max(bottom, pt.point.getY());
+                right = (float) Math.max(right, pt.point.getX());
+            }
+
+            // do not add polygons with too many outer points as they are unlikely to be tables
+            if (poly.size() <= MAX_OUTER_POINT_TOLERANCE) {
+                rectangles.add(new Rectangle(top - AREA_TOLERANCE, left - AREA_TOLERANCE, right - left + 2 * AREA_TOLERANCE, bottom - top + 2 * AREA_TOLERANCE));
+            }
+        }
+        return rectangles;
+    }
+
+
+    private enum Direction {
+        HORIZONTAL,
+        VERTICAL
+    }
+
+    static class PolygonVertex {
+
+        Point2D point;
+        Direction direction;
+
+
+        PolygonVertex(Point2D point, Direction direction) {
+
+            this.direction = direction;
+            this.point = point;
+        }
+
+
+        @Override
+        public boolean equals(Object other) {
+
+            if (this == other) {
+                return true;
+            }
+            if (!(other instanceof PolygonVertex)) {
+                return false;
+            }
+            return this.point.equals(((PolygonVertex) other).point);
+        }
+
+
+        @Override
+        public int hashCode() {
+
+            return this.point.hashCode();
+        }
+
+
+        @Override
+        public String toString() {
+
+            return String.format("%s[point=%s,direction=%s]", this.getClass().getName(), this.point.toString(), this.direction.toString());
+        }
+
+    }
+
+}
@@ -30,8 +30,6 @@ public class TableMergingUtility {
            if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() && !hasTableHeader(consecutiveTable) && outerBoundaryAlignsX(originalTablePageBlock,
                    consecutiveTable)) {
                consecutiveTablesWithSameColCountAndHeaders.add(consecutiveTable);
-            } else {
-                break;
            }
        }
        return Stream.concat(Stream.of(originalTablePageBlock), consecutiveTablesWithSameColCountAndHeaders.stream()).toList();
@@ -3,7 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor.utils;
 import java.util.List;
 import java.util.stream.Collectors;

-
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;

@@ -0,0 +1,44 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// simple implementation of a disjoint-set data structure
+// https://en.wikipedia.org/wiki/Disjoint-set_data_structure
+public class UnionFind<T> {
+
+    Map<T, T> parents = new HashMap<>();
+    Map<T, Integer> numberOfObjects = new HashMap<>();
+
+
+    public T find(T node) {
+
+        if (!parents.containsKey(node)) {
+            parents.put(node, node);
+            numberOfObjects.put(node, 1);
+        }
+        if (!node.equals(parents.get(node))) {
+            parents.put(node, find(parents.get(node)));
+        }
+        return parents.get(node);
+    }
+
+
+    public void union(T node1, T node2) {
+
+        T root1 = find(node1);
+        T root2 = find(node2);
+
+        if (!root1.equals(root2)) {
+            if (numberOfObjects.getOrDefault(root1, 1) < numberOfObjects.getOrDefault(root2, 1)) {
+                parents.put(root1, root2);
+                numberOfObjects.put(root2, numberOfObjects.get(root2) + numberOfObjects.get(root1));
+            } else {
+                parents.put(root2, root1);
+                numberOfObjects.put(root1, numberOfObjects.get(root1) + numberOfObjects.get(root2));
+            }
+        }
+    }
+
+}
+
@@ -37,7 +37,7 @@ public class MessageHandler {

        LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);

-        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS) && layoutParsingRequest.researchDocumentStorageId() == null) {
+        if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.CLARIFYND) && layoutParsingRequest.researchDocumentStorageId() == null) {
            throw new IllegalArgumentException("ResearchDocumentDataStorageId is null!");
        }
        log.info("Layout parsing request received {}", layoutParsingRequest.identifier());
@@ -27,8 +27,10 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
+import com.knecon.fforesight.service.layoutparser.processor.utils.DrawingOptions;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

@@ -46,11 +48,13 @@ public class BdrJsonBuildTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(File file) {

-        return DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.TAAS,
-                file,
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                file.toString()));
+        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.CLARIFYND,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.CLARIFYND,
+                        file,
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        file.toString()));
    }


@@ -111,7 +115,7 @@ public class BdrJsonBuildTest extends AbstractTest {

        try (PDDocument pdDocument = Loader.loadPDF(file); var outputStream = new FileOutputStream(resultingFileName)) {
            PdfDraw.drawDocumentGraph(pdDocument, document);
-            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
+            PdfDraw.drawTextBlock(pdDocument, textBlock, DrawingOptions.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
            pdDocument.save(outputStream);
        }
    }
@@ -33,6 +33,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.tenantcommons.TenantsClient;

@@ -94,11 +95,13 @@ public class HeadlinesGoldStandardIntegrationTest {
        goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));

-        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                pdfFileResource.getFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filePath));
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                        pdfFileResource.getFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        filePath));

        var foundHeadlines = documentGraph.streamAllSubNodes()
                .map(SemanticNode::getHeadline)
@@ -25,10 +25,8 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
    @SneakyThrows
    public void testLayoutParserEndToEnd() {

-        String s = "（";
-        String s1 = "）";
-        prepareStorage("files/Minimal Examples/WrongOrderPage1.pdf");
-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf");
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
        Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info);
    }
@@ -16,6 +16,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
@@ -54,11 +55,13 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
    @SneakyThrows
    private void writeJsons(Path filename) {

-        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                        filename.toFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        filename.toFile().toString()));

        DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
        ObjectMapper mapper = ObjectMapperFactory.create();
@@ -13,6 +13,7 @@ import org.springframework.core.io.ClassPathResource;

 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.DrawingOptions;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;

@@ -70,7 +71,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentTest {
        try (PDDocument pdDocument = Loader.loadPDF(fileResource.getFile())) {
            log.info("drawing document");
            PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
-            PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
+            PdfDraw.drawTextBlock(pdDocument, textBlock, DrawingOptions.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
            log.info("saving document");
            pdDocument.save(tmpFile);
            log.info("saved document");
@@ -12,6 +12,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
@@ -25,7 +26,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
    @SneakyThrows
    public void testViewerDocument() {

-        String fileName = "files/Plenarprotokoll 1 (keine Druchsache!) (1).pdf";
+        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";

        var documentFile = new ClassPathResource(fileName).getFile();
@@ -56,12 +57,14 @@ public class ViewerDocumentTest extends BuildDocumentTest {
                documentFile,
                new ImageServiceResponse(),
                tableResponse,
+                new VisualLayoutParsingResponse(),
                Path.of(fileName).getFileName().toFile().toString());
        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
        LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
-        Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument);
+        Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE, classificationDocument);

        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
    }

 }
+
@@ -29,10 +29,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Im
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
@@ -50,12 +49,6 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @Autowired
    private RedactManagerClassificationService redactManagerClassificationService;

-    @Autowired
-    private CvTableParsingAdapter cvTableParsingAdapter;
-
-    @Autowired
-    private ImageServiceResponseAdapter imageServiceResponseAdapter;
-
    @Autowired
    private SectionsBuilderService sectionsBuilderService;

@@ -63,10 +56,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @SneakyThrows
    public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {

-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
                originDocument,
                new ImageServiceResponse(),
                tableServiceResponse,
+                new VisualLayoutParsingResponse(),
                "document");

        redactManagerClassificationService.classifyDocument(classificationDocument);
@@ -87,11 +81,11 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @Test
    public void tablesToHtmlDebugger() throws IOException {

-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T5 VV-640252-Page16.pdf");

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());

-        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");
+        toHtml(document, "/tmp/T5.html");

    }

@@ -109,6 +103,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @Disabled
    @Test
    public void testScanRotationBorderIsIgnored() throws IOException {

@@ -158,7 +153,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
        assertThat(table.getColCount()).isEqualTo(6);
        assertThat(table.getRowCount()).isEqualTo(13);
        assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@@ -172,10 +167,10 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
        assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
        assertThat(secondTable.getColCount()).isEqualTo(8);
        assertThat(secondTable.getRowCount()).isEqualTo(2);
        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@@ -190,10 +185,10 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
        assertThat(firstTable.getColCount()).isEqualTo(9);
        assertThat(firstTable.getRowCount()).isEqualTo(5);
-        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
        assertThat(secondTable.getColCount()).isEqualTo(9);
        assertThat(secondTable.getRowCount()).isEqualTo(6);
        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
@@ -208,10 +203,10 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        TablePageBlock firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
        assertThat(firstTable.getColCount()).isEqualTo(8);
        assertThat(firstTable.getRowCount()).isEqualTo(1);
-        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        TablePageBlock secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
        assertThat(secondTable.getColCount()).isEqualTo(8);
        assertThat(secondTable.getRowCount()).isEqualTo(6);
        List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@@ -219,7 +214,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


-    @Test // Non-sense test
+    @Test
    public void testDoc56Page170() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
@@ -230,8 +225,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        validateTable(document, 0, 1, 1, 0, 0);
        validateTable(document, 1, 2, 2, 0, 0);
-        validateTable(document, 2, 6, 20, 0, 0);
-        validateTable(document, 3, 7, 31, 0, 0);
+        validateTable(document, 2, 4, 19, 12, 0);
+        validateTable(document, 3, 2, 12, 0, 0);

    }

@@ -579,6 +574,100 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    }


+    @Test
+    public void testT0() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T0 TableWithMergedCells.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 6, 8, 0, 0);
+    }
+
+
+    @Test
+    public void testT1() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T1 MultipleNestedTable.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 3, 3, 0, 0);
+        validateTable(document, 1, 3, 6, 2, 0);
+        validateTable(document, 2, 3, 3, 1, 0);
+        validateTable(document, 3, 3, 3, 0, 0);
+
+    }
+
+
+    @Test
+    public void testT2() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T2 MultipleTables.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 6);
+
+        validateTable(document, 0, 5, 5, 0, 0);
+        validateTable(document, 1, 5, 6, 0, 0);
+        validateTable(document, 2, 5, 5, 0, 0);
+        validateTable(document, 3, 5, 5, 0, 0);
+        validateTable(document, 4, 5, 5, 0, 0);
+        validateTable(document, 5, 5, 5, 0, 0);
+
+    }
+
+
+    @Test
+    public void testT3() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T3 S-Meto_Page29.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 6, 5, 0, 0);
+
+    }
+
+
+    @Test
+    public void testT4() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T4 138 IDD0000261736_Page16.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 5, 8, 1, 0);
+
+    }
+
+
+    @Test
+    public void testT5() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/T5 VV-640252-Page16.pdf");
+
+        ClassificationDocument document = buildClassificationDocument(pdfFileResource.getFile());
+
+        validateTableSize(document, 5);
+        validateTable(document, 0, 1, 1, 0, 0);
+        validateTable(document, 1, 1, 1, 0, 0);
+        validateTable(document, 2, 1, 1, 0, 0);
+        validateTable(document, 3, 1, 1, 0, 0);
+        validateTable(document, 4, 1, 1, 0, 0);
+
+    }
+
+
    @SneakyThrows
    private void toHtml(ClassificationDocument document, String filename) {

@@ -605,7 +694,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {

        TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
        List<List<Cell>> rows = table.getRows();
-        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
+        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().isEmpty()).toList().size();

        for (List<Cell> row : table.getRows()) {
            row.forEach(r -> System.out.println(r.toString()));
@@ -21,7 +21,7 @@ class BodyTextFrameServiceTest extends BuildDocumentTest {

        String filename = "files/211.pdf";
        String outputFilename = "/tmp/" + Path.of(filename).getFileName() + "_MAINBODY.pdf";
-        ClassificationDocument document = parseLayout(filename, LayoutParsingType.TAAS);
+        ClassificationDocument document = parseLayout(filename, LayoutParsingType.CLARIFYND);
        PdfDraw.drawRectanglesPerPage(filename,
                document.getPages().stream().map(page -> List.of(RectangleTransformations.toRectangle2D(page.getBodyTextFrame()))).toList(),
                outputFilename);
@@ -48,7 +48,7 @@ class GapAcrossLinesDetectionServiceTest {


    @Test
-    //@Disabled
+    @Disabled
    @SneakyThrows
    public void testColumnDetection() {

@@ -1,13 +1,17 @@
 package com.knecon.fforesight.service.layoutparser.server.services;

+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.stream.Collectors;

 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;

@@ -21,34 +25,56 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Ta
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.processor.utils.DrawingOptions;
+import com.knecon.fforesight.service.layoutparser.processor.utils.RectangularIntersectionFinder;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
-import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;

 import lombok.SneakyThrows;

 public class RulingCleaningServiceTest extends BuildDocumentTest {

    @Test
-//    @Disabled
+    @Disabled
+    @SneakyThrows
+    public void textRectanglesFromRulingsExtraction() {
+
+        String fileName = "files/SinglePages/T5 VV-640252-Page16.pdf";
+        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_CELLS.pdf";
+        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
+        RulingCleaningService rulingCleaningService = new RulingCleaningService();
+        List<List<Rectangle2D>> rectanglesPerPage = new LinkedList<>();
+        for (PageContents pageContent : pageContents) {
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings());
+            List<Rectangle2D> rects = RectangularIntersectionFinder.find(cleanRulings.getHorizontal(), cleanRulings.getVertical());
+            rectanglesPerPage.add(rects);
+        }
+
+        PdfDraw.drawRectanglesPerPage(fileName, rectanglesPerPage, lineFileName, DrawingOptions.builder().stroke(true).strokeColor(Color.RED).build());
+    }
+
+
+    @Test
+    @Disabled
    @SneakyThrows
    public void textRulingExtraction() {

-        String fileName = "files/211.pdf";
+        String fileName = "files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf";
        String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
        List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
        RulingCleaningService rulingCleaningService = new RulingCleaningService();
-        PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
        List<CleanRulings> cleanRulingsPerPage = new LinkedList<>();
        for (PageContents pageContent : pageContents) {
            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings()));
        }
+        var cleanRulings = cleanRulingsPerPage.stream().map(CleanRulings::getVertical).collect(Collectors.toList());
+        PdfDraw.drawLinesPerPage(fileName, cleanRulings, lineFileName);

    }

@@ -57,9 +83,6 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
    @SneakyThrows
    public void testTableExtraction() {

-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
-        LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
-
        ClassPathResource resource = new ClassPathResource("files");
        List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
                .filter(path -> path.getFileName().toString().endsWith(".pdf"))
@@ -67,8 +90,8 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                .map(Path::toString)
                .toList();

-        for (int i = 0; i < pdfFileNames.size(); i++) {
-            writeJsons(Path.of(pdfFileNames.get(i)));
+        for (String pdfFileName : pdfFileNames) {
+            writeJsons(Path.of(pdfFileName));
        }
    }

@@ -76,25 +99,29 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
    @SneakyThrows
    private void writeJsons(Path filename) {

-        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
-        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                filename.toFile(),
-                new ImageServiceResponse(),
-                new TableServiceResponse(),
-                filename.toFile().toString()));
+        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                        filename.toFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        filename.toFile().toString()));
+        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+                        filename.toFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        filename.toFile().toString()));
        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
-        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
-            String tmpFileNameBefore = "C:/Users/YANNIK~1/AppData/Local/Temp/before." + filename.getFileName().toString();
+        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure())) {
+            String tmpFileNameBefore = "/tmp/before." + filename.getFileName().toString();
            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
                PdfDraw.drawDocumentGraph(pdDocument, documentGraphBefore);
                pdDocument.save(tmpFileNameBefore);
            }
-            String tmpFileNameAfter = "C:/Users/YANNIK~1/AppData/Local/Temp/after." + filename.getFileName().toString();
+            String tmpFileNameAfter = "/tmp/after." + filename.getFileName().toString();
            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
                PdfDraw.drawDocumentGraph(pdDocument, documentGraphAfter);
                pdDocument.save(tmpFileNameAfter);
@@ -105,9 +132,9 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {


    @SneakyThrows
-    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2, String pdfName) {
+    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2) {

-        List listStructure1 = structure1.streamAllEntries()
+        List<Table> listStructure1 = structure1.streamAllEntries()
                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
                .map(DocumentStructure.EntryData::getProperties)
                .map(properties -> {
@@ -117,7 +144,7 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                })
                .toList();

-        List listStructure2 = structure2.streamAllEntries()
+        List<Table> listStructure2 = structure2.streamAllEntries()
                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
                .map(DocumentStructure.EntryData::getProperties)
                .map(properties -> {
@@ -128,8 +155,8 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                .toList();

        for (int i = 0; i < listStructure1.size(); i++) {
-            Table tableNode1 = (Table) listStructure1.get(i);
-            Table tableNode2 = (Table) listStructure2.get(i);
+            Table tableNode1 = listStructure1.get(i);
+            Table tableNode2 = listStructure2.get(i);
            if (tableNode1.getNumberOfRows() != tableNode2.getNumberOfRows() || tableNode1.getNumberOfCols() != tableNode2.getNumberOfCols()) {
                return false;
            }
@@ -51,6 +51,8 @@ public abstract class AbstractTest {
    private RabbitTemplate rabbitTemplate;

    protected final static String ORIGIN_FILE_ID = "origin";
+
+    protected final static String VISUAL_LAYOUT_FILE = "visual";
    protected final static String TABLE_FILE_ID = "table";
    protected final static String IMAGE_FILE_ID = "image";
    protected final static String STRUCTURE_FILE_ID = "structure";
@@ -65,7 +67,7 @@ public abstract class AbstractTest {
    protected LayoutParsingRequest buildStandardLayoutParsingRequest() {

        return LayoutParsingRequest.builder()
-                .layoutParsingType(LayoutParsingType.REDACT_MANAGER)
+                .layoutParsingType(LayoutParsingType.REDACT_MANAGER_OLD)
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
@@ -96,7 +98,7 @@ public abstract class AbstractTest {
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(String file) {

-        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json");
+        return prepareStorage(file, "cv_table_parsing_response/empty.json", "image_service_response/empty.json", "visual_layout_parsing_response/empty.json");
    }


@@ -104,7 +106,7 @@ public abstract class AbstractTest {
    protected LayoutParsingRequest prepareStorage(InputStream fileInputStream) {

        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileInputStream);
-        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
    }


@@ -116,6 +118,7 @@ public abstract class AbstractTest {
                .originFileStorageId(ORIGIN_FILE_ID)
                .tablesFileStorageId(Optional.of(TABLE_FILE_ID))
                .imagesFileStorageId(Optional.of(IMAGE_FILE_ID))
+                .visualLayoutParsingFileId(Optional.of(VISUAL_LAYOUT_FILE))
                .structureFileStorageId(STRUCTURE_FILE_ID)
                .textBlockFileStorageId(TEXT_FILE_ID)
                .positionBlockFileStorageId(POSITION_FILE_ID)
@@ -137,6 +140,21 @@ public abstract class AbstractTest {
    }


+    @SneakyThrows
+    protected LayoutParsingRequest prepareStorage(String file, String cvServiceResponseFile, String imageInfoFile, String visualLayoutParsingResponseFile) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+        ClassPathResource imageInfoFileResource = new ClassPathResource(imageInfoFile);
+        ClassPathResource visualLayoutParsingResponseResource = new ClassPathResource(visualLayoutParsingResponseFile);
+
+        return prepareStorage(pdfFileResource.getInputStream(),
+                cvServiceResponseFileResource.getInputStream(),
+                imageInfoFileResource.getInputStream(),
+                visualLayoutParsingResponseResource.getInputStream());
+    }
+
+
    @SneakyThrows
    protected LayoutParsingRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream, InputStream imageInfoStream) {

@@ -144,7 +162,22 @@ public abstract class AbstractTest {
        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);

-        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER);
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
+    }
+
+
+    @SneakyThrows
+    protected LayoutParsingRequest prepareStorage(InputStream fileStream,
+                                                  InputStream cvServiceResponseFileStream,
+                                                  InputStream imageInfoStream,
+                                                  InputStream visualLayoutParsingResponseFileStream) {
+
+        storageService.storeObject(TenantContext.getTenantId(), IMAGE_FILE_ID, imageInfoStream);
+        storageService.storeObject(TenantContext.getTenantId(), TABLE_FILE_ID, cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(), ORIGIN_FILE_ID, fileStream);
+        storageService.storeObject(TenantContext.getTenantId(), VISUAL_LAYOUT_FILE, visualLayoutParsingResponseFileStream);
+
+        return buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER_OLD);
    }


@@ -10,6 +10,7 @@ import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipelin
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;

 import lombok.SneakyThrows;
@@ -25,14 +26,19 @@ public abstract class BuildDocumentTest extends AbstractTest {

        File fileResource = new ClassPathResource(filename).getFile();
        prepareStorage(filename);
-        return layoutParsingPipeline.parseLayout(layoutParsingType, fileResource, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse(), filename);
+        return layoutParsingPipeline.parseLayout(layoutParsingType,
+                fileResource,
+                layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
+                new TableServiceResponse(),
+                new VisualLayoutParsingResponse(),
+                filename);
    }


    @SneakyThrows
    protected Document buildGraph(String filename) {

-        return buildGraph(filename, LayoutParsingType.REDACT_MANAGER);
+        return buildGraph(filename, LayoutParsingType.REDACT_MANAGER_OLD);
    }


@@ -45,7 +51,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
            prepareStorage(filename);
        }

-        return DocumentGraphFactory.buildDocumentGraph(parseLayout(filename, layoutParsingType));
+        return DocumentGraphFactory.buildDocumentGraph(layoutParsingType, parseLayout(filename, layoutParsingType));
    }

 }
@@ -24,20 +24,31 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.utils.DrawingOptions;
 import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;

-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.experimental.UtilityClass;

@UtilityClass
 public class PdfDraw {

+    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName, DrawingOptions options) throws IOException {
+
+        ClassPathResource pdfResource = new ClassPathResource(filename);
+        try (PDDocument pdDocument = Loader.loadPDF(pdfResource.getFile()); var out = new FileOutputStream(tmpFileName)) {
+            for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
+                        pageNumber,
+                        rectanglesPerPage.get(pageNumber - 1),
+                        options);
+            }
+            pdDocument.save(out);
+        }
+
+    }
+
    public static void drawRectanglesPerPage(String filename, List<List<Rectangle2D>> rectanglesPerPage, String tmpFileName) throws IOException {

        ClassPathResource pdfResource = new ClassPathResource(filename);
@@ -46,7 +57,7 @@ public class PdfDraw {
                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
                        pageNumber,
                        rectanglesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
+                        DrawingOptions.builder().stroke(true).build());
            }
            pdDocument.save(out);
        }
@@ -62,13 +73,13 @@ public class PdfDraw {
                var rectanglesOnPage = rectanglesPerPage.get(pageNumber - 1);
                for (int lineNumber = 0; lineNumber < rectanglesOnPage.size(); lineNumber++) {
                    var rectanglesInLine = rectanglesOnPage.get(lineNumber);
-                    PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, rectanglesInLine, PdfVisualisationUtility.Options.builder().stroke(true).build());
+                    PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, rectanglesInLine, DrawingOptions.builder().stroke(true).build());
                    double y = Math.min(rectanglesInLine.get(0).getMinY(), rectanglesInLine.get(0).getMaxY());
                    PdfVisualisationUtility.drawText(String.format("%d", lineNumber),
                            pdDocument,
                            new Point2D.Double(rectanglesInLine.get(0).getX() - (5 + (5 * countNumberOfDigits(lineNumber))), y + 2),
                            pageNumber,
-                            PdfVisualisationUtility.Options.builder().stroke(true).build());
+                            DrawingOptions.builder().stroke(true).build());
                }
            }
            pdDocument.save(out);
@@ -99,20 +110,20 @@ public class PdfDraw {

    public static void drawNode(PDDocument document, DocumentTree.Entry entry) {

-        Options options = buildStandardOptionsForNodes(entry);
+        DrawingOptions options = buildStandardOptionsForNodes(entry);

        drawBBoxAndLabelAndNumberOnPage(document, entry, options);

    }


-    public static void drawTextBlock(PDDocument document, TextBlock textBlock, Options options) {
+    public static void drawTextBlock(PDDocument document, TextBlock textBlock, DrawingOptions options) {

        textBlock.getAtomicTextBlocks().forEach(atb -> drawAtomicTextBlock(document, atb, options));
    }


-    public static void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, Options options) {
+    public static void drawAtomicTextBlock(PDDocument document, AtomicTextBlock atomicTextBlock, DrawingOptions options) {

        drawRectangle2DList(document, atomicTextBlock.getPage().getNumber(), atomicTextBlock.getPositions().stream().toList(), options);

@@ -120,7 +131,7 @@ public class PdfDraw {


    @SneakyThrows
-    private static void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, Options options, boolean rotate) {
+    private static void drawText(String string, PDDocument document, Point2D location, Integer pageNumber, DrawingOptions options, boolean rotate) {

        var pdPage = document.getPage(pageNumber - 1);
        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
@@ -142,14 +153,14 @@ public class PdfDraw {


    @SneakyThrows
-    public static void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, Options options) {
+    public static void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, DrawingOptions options) {

        var pdPage = document.getPage(pageNumber - 1);
        drawRectangle2DList(document, rectCollection, options, pdPage);
    }


-    private static void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, Options options, PDPage pdPage) throws IOException {
+    private static void drawRectangle2DList(PDDocument document, List<Rectangle2D> rectCollection, DrawingOptions options, PDPage pdPage) throws IOException {

        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);

@@ -181,12 +192,12 @@ public class PdfDraw {
 //                PdfVisualisationUtility.drawLine2DList(pdDocument,
 //                        pageNumber,
 //                        list.get(pageNumber - 1),
-//                        PdfVisualisationUtility.Options.builder().stroke(true).build());
+//                        PdfVisualisationUtility.DrawingOptions.builder().stroke(true).build());
                PdfVisualisationUtility.drawRectangle2DList(pdDocument,
                        pageNumber,
                        rectanglesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().stroke(true).build());
-                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), PdfVisualisationUtility.Options.builder().stroke(true).build());
+                        DrawingOptions.builder().stroke(true).build());
+                PdfVisualisationUtility.drawRectangle2DList(pdDocument, pageNumber, list.get(pageNumber - 1), DrawingOptions.builder().stroke(true).build());
            }
            pdDocument.save(out);
        }
@@ -202,35 +213,18 @@ public class PdfDraw {
                PdfVisualisationUtility.drawLine2DList(pdDocument,
                        pageNumber,
                        linesPerPage.get(pageNumber - 1),
-                        PdfVisualisationUtility.Options.builder().strokeColor(Color.RED).stroke(true).build());
+                        DrawingOptions.builder().strokeColor(Color.RED).stroke(true).build());
            }
            pdDocument.save(out);
        }
    }


-    @Builder
-    @AllArgsConstructor
-    @Getter
-    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    public static class Options {
-
-        boolean stroke;
-        @Builder.Default
-        Color strokeColor = Color.BLACK;
-        @Builder.Default
-        float strokeWidth = 1f;
-
-        boolean fill;
-        @Builder.Default
-        Color fillColor = Color.BLACK;
-
-    }


-    private static Options buildStandardOptionsForNodes(DocumentTree.Entry entry) {
+    private static DrawingOptions buildStandardOptionsForNodes(DocumentTree.Entry entry) {

-        return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
+        return DrawingOptions.builder().stroke(true).strokeColor(switch (entry.getType()) {
            case DOCUMENT -> Color.LIGHT_GRAY;
            case HEADER, FOOTER -> Color.GREEN;
            case PARAGRAPH -> Color.BLUE;
@@ -243,7 +237,7 @@ public class PdfDraw {
    }


-    private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, Options options) {
+    private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, DocumentTree.Entry entry, DrawingOptions options) {

        Map<Page, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
        for (Page page : rectanglesPerPage.keySet()) {
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Dominique Eifländer	79239b751d	RED-7141: Implemented docstrum layout parsing	2024-03-06 11:18:40 +01:00
Yannik Hampe	f146beeb44	Merge branch 'RED-8481-hotfix' into 'main' RED-8481: Use visual layout parsing to detect signatures See merge request fforesight/layout-parser!106	2024-02-29 09:39:17 +01:00
yhampe	f8a4ccfff0	Merge remote-tracking branch 'origin/RED-8481-hotfix' into RED-8481-hotfix	2024-02-29 09:22:41 +01:00
yhampe	a6ba501fa8	RED-8481: Use visual layout parsing to detect signatures fixed some nullpointer errors	2024-02-29 09:22:27 +01:00
yhampe	7dfb3b2b52	RED-8481: Use visual layout parsing to detect signatures fixed some nullpointer errors	2024-02-29 09:21:47 +01:00
Kilian Schüttler	c324d3815e	Merge branch 'RED-8550-pre-backup' into 'main' RED-8550: Faulty table recognition and text duplication leads to huge sections See merge request fforesight/layout-parser!105	2024-02-28 16:13:56 +01:00
Maverick Studer	74f55a5cbf	RED-8550: Faulty table recognition and text duplication leads to huge sections	2024-02-28 16:13:56 +01:00
Kilian Schüttler	e7bf607663	Merge branch 'hotfix' into 'main' hotfix: double viewerdoc writes in rare cases lead to some contentstreams not being written See merge request fforesight/layout-parser!104	2024-02-26 12:32:14 +01:00
Kilian Schuettler	f4d789311c	hotfix: double viewerdoc writes in rare cases lead to some contentstreams not being written	2024-02-26 12:24:15 +01:00
Yannik Hampe	9817eae897	Merge branch 'RED-8481-fix' into 'main' Red 8481 fix See merge request fforesight/layout-parser!103	2024-02-23 14:08:57 +01:00
yhampe	477f6af886	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results checkstyle	2024-02-23 14:02:53 +01:00
yhampe	2c171b6a9e	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results codestyle	2024-02-23 13:55:11 +01:00
yhampe	71477dabde	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results codestyle	2024-02-23 12:46:51 +01:00
yhampe	a927cbd9dc	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results fixed tests	2024-02-23 12:38:05 +01:00
yhampe	a1521877d7	RED-8481: Use visual layout parsing to detect signatures added a new layer for visual parsing results added a source label to image properties to enable rules	2024-02-23 12:20:11 +01:00
Maverick Studer	f4b6386e1c	Merge branch 'RED-8550-testing' into 'main' RED-8550: Faulty table recognition and text duplication leads to huge sections See merge request fforesight/layout-parser!102	2024-02-21 13:54:30 +01:00
Maverick Studer	1d64028158	RED-8550: Faulty table recognition and text duplication leads to huge sections	2024-02-21 13:54:30 +01:00
Yannik Hampe	0979a267d4	Merge branch 'RED-8481' into 'main' RED-8481: Use visual layout parsing to detect signatures See merge request fforesight/layout-parser!101	2024-02-15 13:07:06 +01:00
yhampe	cc77d19500	RED-8481: Use visual layout parsing to detect signatures addressed review comments	2024-02-15 13:01:30 +01:00
yhampe	fa048b2fe0	RED-8481: Use visual layout parsing to detect signatures addressed review comments	2024-02-15 12:19:26 +01:00
yhampe	bdf1161c91	RED-8481: Use visual layout parsing to detect signatures addressed review comments	2024-02-15 12:12:23 +01:00
yhampe	b4a225144d	RED-8481: Use visual layout parsing to detect signatures working on failing tests	2024-02-15 10:16:07 +01:00
yhampe	903b1c1fd4	RED-8481: Use visual layout parsing to detect signatures fixed failing tests because of null pointer	2024-02-15 09:27:07 +01:00
yhampe	c3e7582ee3	RED-8481: Use visual layout parsing to detect signatures fixed failing tests because of null pointer	2024-02-14 12:33:36 +01:00
yhampe	cfc5db45cd	RED-8481: Use visual layout parsing to detect signatures fixed failing tests because of null pointer	2024-02-14 12:24:32 +01:00
yhampe	fbd0196719	RED-8481: Use visual layout parsing to detect signatures implemented visuallayoutparsingresult	2024-02-14 12:16:37 +01:00